model2data 1.16.4__tar.gz → 1.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.16.4/model2data.egg-info → model2data-1.18.0}/PKG-INFO +1 -1
- {model2data-1.16.4 → model2data-1.18.0}/README.md +11 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/__init__.py +20 -0
- model2data-1.18.0/model2data/metrics/slices.py +343 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/sql.py +39 -10
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/values.py +30 -20
- {model2data-1.16.4 → model2data-1.18.0/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.16.4 → model2data-1.18.0}/model2data.egg-info/SOURCES.txt +2 -0
- {model2data-1.16.4 → model2data-1.18.0}/pyproject.toml +1 -1
- model2data-1.18.0/tests/test_metrics_slices.py +400 -0
- {model2data-1.16.4 → model2data-1.18.0}/LICENSE +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/README_PYPI.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/LICENSE +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/VENDORED.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/base.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/column.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/index.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/project.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/table.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/constants.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/database.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/column.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/common.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/index.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/project.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/records.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/table.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/exceptions.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/parser/parser.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/base.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/_vendor/pydbml/tools.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/cli.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/hint_tests.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/naming.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/project.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/hint_macros/model2data_when_tests.sql +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/history_macros/model2data_history_tests.sql +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/dbt/tests.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/defects/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/defects/apply.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/defects/checks.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/defects/presets.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/defects/report.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/core.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/days.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/faker.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/hints.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/history.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/kinds.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/options.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/parents.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/relationships.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/timeline.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/generate/when.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/guide/setup.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/guide/triage.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/guide/tune.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/check.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/columns.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/dbt.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/expression.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/graph.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/infer.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/lightdash.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/ossie.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/reader.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/semantic.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/metrics/types.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/_yaml.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/dbml.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/document.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/dump.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/engine.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/errors.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/reader.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/types.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/model/validate.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/output.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/parse/__init__.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/parse/dbml.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/PUBLISHING.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/README.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/examples/coffee_webshop.metrics.yml +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/metrics/README.md +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/metrics/metrics.schema.json +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/spec/model.schema.json +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data/utils.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/setup.cfg +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_cli.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_cli_model.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_column_time_hints.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_days.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbml_parser.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbt_integration.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbt_naming.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbt_project.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_dbt_tests.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_defects.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_defects_dbt.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_defects_replay.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_defects_report.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_defects_review.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_determinism.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_distributions.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_examples_model.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_fast_paths.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_generation.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_guide.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_hint_tests.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_history.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_issue_lines.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_lone_country.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_cli.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_dbt.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_expression.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_lightdash.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_ossie.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_reader.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_semantic.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_spec.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_metrics_values.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_model.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_model_dbml.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_model_edges.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_model_messages.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_model_vocabulary.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_options.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_parent_dates.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_release_stress.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_reproducibility.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_row_identity.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_shaping.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_spec_alignment.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_spec_schema.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_table_seeds.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_timeline.py +0 -0
- {model2data-1.16.4 → model2data-1.18.0}/tests/test_when.py +0 -0
|
@@ -353,6 +353,17 @@ newest release. Pinned to an older dbt-core? model2data 0.5.x supports down to 1
|
|
|
353
353
|
- A reference onto a column that is not a key is allowed with a warning; its values are drawn
|
|
354
354
|
from the ones the parent column holds.
|
|
355
355
|
|
|
356
|
+
## In the wild
|
|
357
|
+
|
|
358
|
+
**[Shaper](https://taleshape.com) by Taleshape.** Jorin Vogel's demo starts from one
|
|
359
|
+
`.model2data.yml` file for a network of eight outpatient clinics: 1,500 patients, 6,000
|
|
360
|
+
appointments and a dbt project, all in DuckDB. Shaper reads the same DuckDB file, and an AI agent wrote the dashboard on
|
|
361
|
+
top of it. [Blog post](https://taleshape.com/blog/dashboards-as-code-for-your-dbt-project/),
|
|
362
|
+
[live dashboard](https://demo.taleshape.com/view/clinic-overview),
|
|
363
|
+
[repo](https://github.com/taleshape-com/model2data-shaper-demo).
|
|
364
|
+
|
|
365
|
+
Built something on model2data? Open an issue and we'll add it here.
|
|
366
|
+
|
|
356
367
|
---
|
|
357
368
|
|
|
358
369
|
## Project status
|
|
@@ -10,6 +10,10 @@ generator never reads it: a model generates the same data with or without one.
|
|
|
10
10
|
>>> found = metrics.load("coffee_webshop.metrics.yml", model) # MetricsError on errors
|
|
11
11
|
>>> semantic = metrics.resolve(model, found) # entities, relationships, dimensions, metrics
|
|
12
12
|
>>> values = metrics.known_values(semantic, frames) # over a run's generated tables
|
|
13
|
+
>>> narrowed = metrics.filtered(semantic, {"orders.status": ["paid"]})
|
|
14
|
+
>>> metrics.known_values(narrowed.semantic, frames) # the same, on those rows
|
|
15
|
+
>>> metrics.breakdowns(semantic, frames, [("revenue", "customers.country")])
|
|
16
|
+
>>> metrics.by_period(semantic, frames, "week") # any grain, ISO weeks
|
|
13
17
|
>>> export = metrics.to_ossie(semantic) # Apache Ossie 0.1.1
|
|
14
18
|
>>> export.to_yaml(), export.lossiness
|
|
15
19
|
>>> metrics.to_lightdash(semantic).to_yaml() # Lightdash meta, dbt YAML
|
|
@@ -44,6 +48,15 @@ from model2data.metrics.semantic import (
|
|
|
44
48
|
SemanticModel,
|
|
45
49
|
resolve,
|
|
46
50
|
)
|
|
51
|
+
from model2data.metrics.slices import (
|
|
52
|
+
GRAINS,
|
|
53
|
+
Breakdown,
|
|
54
|
+
Filtered,
|
|
55
|
+
Member,
|
|
56
|
+
breakdowns,
|
|
57
|
+
by_period,
|
|
58
|
+
filtered,
|
|
59
|
+
)
|
|
47
60
|
from model2data.metrics.types import (
|
|
48
61
|
AGGREGATIONS,
|
|
49
62
|
FORMATS,
|
|
@@ -60,15 +73,18 @@ from model2data.metrics.values import KnownValue, KnownValues, known_values
|
|
|
60
73
|
__all__ = [
|
|
61
74
|
"AGGREGATIONS",
|
|
62
75
|
"FORMATS",
|
|
76
|
+
"GRAINS",
|
|
63
77
|
"OPERATORS",
|
|
64
78
|
"OSSIE_VERSION",
|
|
65
79
|
"SCHEMA_URL",
|
|
66
80
|
"SPEC_VERSION",
|
|
81
|
+
"Breakdown",
|
|
67
82
|
"Condition",
|
|
68
83
|
"Dimension",
|
|
69
84
|
"DimensionOverride",
|
|
70
85
|
"Entity",
|
|
71
86
|
"ExpressionError",
|
|
87
|
+
"Filtered",
|
|
72
88
|
"Graph",
|
|
73
89
|
"Join",
|
|
74
90
|
"KnownValue",
|
|
@@ -76,6 +92,7 @@ __all__ = [
|
|
|
76
92
|
"LightdashExport",
|
|
77
93
|
"Loss",
|
|
78
94
|
"Measure",
|
|
95
|
+
"Member",
|
|
79
96
|
"Metric",
|
|
80
97
|
"Metrics",
|
|
81
98
|
"MetricsError",
|
|
@@ -90,6 +107,9 @@ __all__ = [
|
|
|
90
107
|
"from_dict",
|
|
91
108
|
"inferred_metrics",
|
|
92
109
|
"is_metrics_file",
|
|
110
|
+
"breakdowns",
|
|
111
|
+
"by_period",
|
|
112
|
+
"filtered",
|
|
93
113
|
"known_values",
|
|
94
114
|
"load",
|
|
95
115
|
"metric_test_sql",
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""A run's metrics narrowed to members, broken down by a dimension, or per period.
|
|
2
|
+
|
|
3
|
+
All three are the same metrics over the same tables, so they cannot come to
|
|
4
|
+
disagree with the totals in `known_values`:
|
|
5
|
+
|
|
6
|
+
- `filtered` adds a condition to each metric's own filter and hands back a
|
|
7
|
+
semantic model, which `known_values` then computes as usual: per month, and
|
|
8
|
+
through ratios and derived metrics, without a second way of doing either.
|
|
9
|
+
- `breakdowns` runs a metric's own query with a `group by`. A ratio or derived
|
|
10
|
+
metric is computed per member from its inputs per member, exactly as its total
|
|
11
|
+
is from theirs.
|
|
12
|
+
- `by_period` is `by_month` at another grain: each period is grouped in the
|
|
13
|
+
metric's own SQL, never added up from smaller periods, which would be wrong for
|
|
14
|
+
an average, a median or a distinct count.
|
|
15
|
+
|
|
16
|
+
A dimension applies to a metric when the metric's table reaches the dimension's
|
|
17
|
+
table along exactly one many-to-one path, or is that table. Anything else would
|
|
18
|
+
repeat the metric's rows or guess between paths, so `filtered` lists the metrics
|
|
19
|
+
it could not narrow, and a `Breakdown` says why it has no members. A ratio or
|
|
20
|
+
derived metric applies only if all of its inputs do.
|
|
21
|
+
|
|
22
|
+
The cap on members follows what can be added up: the members past `limit`
|
|
23
|
+
are summed as `other` for a sum or a count, and left out, with a `note`, for
|
|
24
|
+
anything else, since an average of the rest is not the rest of an average.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import datetime as dt
|
|
30
|
+
from collections.abc import Mapping, Sequence
|
|
31
|
+
from dataclasses import dataclass, field, replace
|
|
32
|
+
from decimal import Context, Decimal, InvalidOperation, localcontext
|
|
33
|
+
from typing import Any, Optional
|
|
34
|
+
|
|
35
|
+
import pandas as pd
|
|
36
|
+
|
|
37
|
+
from model2data.metrics.columns import ColumnKind, column_kind, find_column
|
|
38
|
+
from model2data.metrics.graph import Graph
|
|
39
|
+
from model2data.metrics.semantic import ResolvedMetric, SemanticModel
|
|
40
|
+
from model2data.metrics.sql import columns_read, query
|
|
41
|
+
from model2data.metrics.types import Condition, Where
|
|
42
|
+
from model2data.metrics.values import (
|
|
43
|
+
Number,
|
|
44
|
+
_assemble,
|
|
45
|
+
_empty,
|
|
46
|
+
_evaluate,
|
|
47
|
+
_exact,
|
|
48
|
+
_rounded,
|
|
49
|
+
_tables,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
DEFAULT_MEMBERS = 12
|
|
53
|
+
# Weeks are ISO weeks, starting on Monday.
|
|
54
|
+
GRAINS: tuple[str, ...] = ("day", "week", "month", "quarter", "year")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class Filtered:
|
|
59
|
+
"""`semantic` with its metrics narrowed; `unfiltered` names those a dimension did
|
|
60
|
+
not apply to, whose values are not narrowed and should not be read as if they were."""
|
|
61
|
+
|
|
62
|
+
semantic: SemanticModel
|
|
63
|
+
unfiltered: tuple[str, ...] = ()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class Member:
|
|
68
|
+
"""One member of a dimension and the metric's value there. `member` is the column's
|
|
69
|
+
text, None for rows where it is null."""
|
|
70
|
+
|
|
71
|
+
member: Optional[str]
|
|
72
|
+
value: Optional[Number]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class Breakdown:
|
|
77
|
+
metric: str
|
|
78
|
+
by: str
|
|
79
|
+
members: list[Member] = field(default_factory=list)
|
|
80
|
+
# The members past the cap, summed; None when there are none or they cannot be added.
|
|
81
|
+
other: Optional[Number] = None
|
|
82
|
+
# How many members are not in `members`, summed into `other` or left out.
|
|
83
|
+
hidden: int = 0
|
|
84
|
+
# Why there is less here than a person would expect, in words.
|
|
85
|
+
note: Optional[str] = None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _column(semantic: SemanticModel, path: str) -> tuple[str, ColumnKind]:
|
|
89
|
+
table, column = find_column(semantic.model, path)
|
|
90
|
+
if table is None or column is None:
|
|
91
|
+
raise ValueError(f"no column {path!r} in the model")
|
|
92
|
+
return table, column_kind(semantic.model, column)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _reaching(semantic: SemanticModel, metric: ResolvedMetric, table: str) -> Optional[dict]:
|
|
96
|
+
"""`metric`'s joins with `table` among them, or None when it has no one way there."""
|
|
97
|
+
assert metric.table is not None
|
|
98
|
+
if table == metric.table or table in metric.joins:
|
|
99
|
+
return metric.joins
|
|
100
|
+
paths = Graph(semantic.model).paths(metric.table, table, limit=2)
|
|
101
|
+
return {**metric.joins, table: paths[0]} if len(paths) == 1 else None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _value(value: Any, kind: ColumnKind, path: str) -> Any:
|
|
105
|
+
"""A member as a value of the column's kind. Checked because it may come from a
|
|
106
|
+
request, and a number or a date is written into the query without quotes."""
|
|
107
|
+
try:
|
|
108
|
+
if kind == "integer":
|
|
109
|
+
return int(str(value))
|
|
110
|
+
if kind == "decimal":
|
|
111
|
+
number = Decimal(str(value))
|
|
112
|
+
if not number.is_finite():
|
|
113
|
+
raise ValueError(value)
|
|
114
|
+
return number
|
|
115
|
+
if kind == "date":
|
|
116
|
+
return dt.date.fromisoformat(str(value)).isoformat()
|
|
117
|
+
if kind == "timestamp":
|
|
118
|
+
return dt.datetime.fromisoformat(str(value)).isoformat(sep=" ")
|
|
119
|
+
if kind == "boolean":
|
|
120
|
+
text = str(value).lower()
|
|
121
|
+
if text not in ("true", "false"):
|
|
122
|
+
raise ValueError(value)
|
|
123
|
+
return text == "true"
|
|
124
|
+
except (ValueError, InvalidOperation) as error:
|
|
125
|
+
raise ValueError(f"{value!r} is not a value of {path} ({kind})") from error
|
|
126
|
+
return str(value)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _keeping(path: str, members: Sequence[Any], kind: ColumnKind) -> Where:
|
|
130
|
+
"""The filter for rows whose `path` is one of `members`, None among them meaning null."""
|
|
131
|
+
values = [_value(member, kind, path) for member in members if member is not None]
|
|
132
|
+
alternatives = [Where(terms=[Condition(path, {"in": values})])] if values else []
|
|
133
|
+
if len(values) < len(members):
|
|
134
|
+
alternatives.append(Where(terms=[Condition(path, {"is_null": True})]))
|
|
135
|
+
return Where(terms=alternatives, any=True)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _within(path: str, start: Any, end: Any, kind: ColumnKind) -> Where:
|
|
139
|
+
"""The filter for rows whose date or timestamp `path` falls from `start` to `end`.
|
|
140
|
+
|
|
141
|
+
Both ends are included and either may be None. A timestamp's `end` given as a day
|
|
142
|
+
means all of that day, which is what a date picker's last day means.
|
|
143
|
+
"""
|
|
144
|
+
if kind not in ("date", "timestamp"):
|
|
145
|
+
raise ValueError(f"{path} is not a date or a timestamp ({kind})")
|
|
146
|
+
operators: dict[str, Any] = {}
|
|
147
|
+
if start is not None:
|
|
148
|
+
operators["gte"] = _value(start, kind, path)
|
|
149
|
+
if end is not None:
|
|
150
|
+
written = str(end)
|
|
151
|
+
if kind == "timestamp" and len(written) == 10:
|
|
152
|
+
day = dt.date.fromisoformat(_value(written, "date", path))
|
|
153
|
+
operators["lt"] = (day + dt.timedelta(days=1)).isoformat()
|
|
154
|
+
else:
|
|
155
|
+
operators["lte"] = _value(end, kind, path)
|
|
156
|
+
return Where(terms=[Where(terms=[Condition(path, operators)])])
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def filtered(
|
|
160
|
+
semantic: SemanticModel,
|
|
161
|
+
members: Mapping[str, Sequence[Any]],
|
|
162
|
+
*,
|
|
163
|
+
ranges: Optional[Mapping[str, tuple[Any, Any]]] = None,
|
|
164
|
+
) -> Filtered:
|
|
165
|
+
"""Every metric narrowed to rows whose dimension is one of its members.
|
|
166
|
+
|
|
167
|
+
`members` maps a column path to the values to keep, None for null. `ranges` maps a
|
|
168
|
+
date or timestamp column to its `(start, end)`, both included, None for open. All of
|
|
169
|
+
them together narrow as `and`. A dimension with no members keeps everything, and so
|
|
170
|
+
does a range open at both ends.
|
|
171
|
+
"""
|
|
172
|
+
wanted = {path: kept for path, kept in members.items() if len(kept) > 0}
|
|
173
|
+
terms = [
|
|
174
|
+
(path.rpartition(".")[0], _keeping(path, kept, _column(semantic, path)[1]))
|
|
175
|
+
for path, kept in wanted.items()
|
|
176
|
+
]
|
|
177
|
+
terms += [
|
|
178
|
+
(path.rpartition(".")[0], _within(path, start, end, _column(semantic, path)[1]))
|
|
179
|
+
for path, (start, end) in (ranges or {}).items()
|
|
180
|
+
if start is not None or end is not None
|
|
181
|
+
]
|
|
182
|
+
metrics = dict(semantic.metrics)
|
|
183
|
+
missed: set[str] = set()
|
|
184
|
+
for name, metric in semantic.metrics.items():
|
|
185
|
+
if metric.kind not in ("simple", "count"):
|
|
186
|
+
continue
|
|
187
|
+
joins = metric.joins
|
|
188
|
+
for table, _ in terms:
|
|
189
|
+
joins = _reaching(semantic, replace(metric, joins=joins), table)
|
|
190
|
+
if joins is None:
|
|
191
|
+
break
|
|
192
|
+
if joins is None:
|
|
193
|
+
missed.add(name)
|
|
194
|
+
continue
|
|
195
|
+
if terms:
|
|
196
|
+
kept = metric.where.terms if metric.where is not None else []
|
|
197
|
+
where = Where(terms=[*kept, *(term for _, term in terms)])
|
|
198
|
+
metrics[name] = replace(metric, joins=joins, where=where)
|
|
199
|
+
unfiltered = [name for name in semantic.metrics if missed & set(semantic.simple_inputs(name))]
|
|
200
|
+
return Filtered(replace(semantic, metrics=metrics), tuple(unfiltered))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _additive(metric: ResolvedMetric) -> bool:
|
|
204
|
+
return metric.kind == "count" or metric.agg in ("sum", "count")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _sort_key(member: Member) -> tuple:
|
|
208
|
+
value = member.value
|
|
209
|
+
return (value is None, -value if value is not None else 0, member.member is None, member.member)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def breakdowns(
|
|
213
|
+
semantic: SemanticModel,
|
|
214
|
+
frames: Mapping[str, pd.DataFrame],
|
|
215
|
+
wanted: Sequence[tuple[str, str]],
|
|
216
|
+
*,
|
|
217
|
+
limit: int = DEFAULT_MEMBERS,
|
|
218
|
+
) -> list[Breakdown]:
|
|
219
|
+
"""Each `(metric, column path)` of `wanted`, its members largest first."""
|
|
220
|
+
out: dict[tuple[str, str], Breakdown] = {}
|
|
221
|
+
# What to query: a simple metric and dimension, with the joins that reach it.
|
|
222
|
+
views: dict[tuple[str, str], ResolvedMetric] = {}
|
|
223
|
+
for name, by in dict.fromkeys(wanted):
|
|
224
|
+
if name not in semantic.metrics:
|
|
225
|
+
raise ValueError(f"no metric {name!r}")
|
|
226
|
+
table = _column(semantic, by)[0]
|
|
227
|
+
inputs = semantic.simple_inputs(name)
|
|
228
|
+
joins = {dep: _reaching(semantic, semantic.metrics[dep], table) for dep in inputs}
|
|
229
|
+
if any(item is None for item in joins.values()):
|
|
230
|
+
out[(name, by)] = Breakdown(
|
|
231
|
+
name, by, note=f"{name} has no single many-to-one path to {table}"
|
|
232
|
+
)
|
|
233
|
+
continue
|
|
234
|
+
out[(name, by)] = Breakdown(name, by)
|
|
235
|
+
for dep in inputs:
|
|
236
|
+
views[(dep, by)] = replace(semantic.metrics[dep], joins=joins[dep] or {})
|
|
237
|
+
|
|
238
|
+
def seen(dep: str, by: str) -> SemanticModel:
|
|
239
|
+
return replace(semantic, metrics={**semantic.metrics, dep: views[(dep, by)]})
|
|
240
|
+
|
|
241
|
+
read: dict[str, set[str]] = {}
|
|
242
|
+
for dep, by in views:
|
|
243
|
+
for table, columns in columns_read(seen(dep, by), dep, by=by).items():
|
|
244
|
+
read.setdefault(table, set()).update(columns)
|
|
245
|
+
raw: dict[tuple[str, str], dict[Optional[str], Any]] = {}
|
|
246
|
+
if views:
|
|
247
|
+
with _tables(semantic, frames, read) as (connection, relation):
|
|
248
|
+
for dep, by in views:
|
|
249
|
+
sql = query(seen(dep, by), dep, relation, by=by)
|
|
250
|
+
raw[(dep, by)] = dict(connection.execute(sql).fetchall())
|
|
251
|
+
|
|
252
|
+
for breakdown in out.values():
|
|
253
|
+
if breakdown.note is None:
|
|
254
|
+
_fill(semantic, breakdown, raw, limit)
|
|
255
|
+
return [out[key] for key in dict.fromkeys(wanted)]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _fill(
|
|
259
|
+
semantic: SemanticModel,
|
|
260
|
+
breakdown: Breakdown,
|
|
261
|
+
raw: dict[tuple[str, str], dict[Optional[str], Any]],
|
|
262
|
+
limit: int,
|
|
263
|
+
) -> None:
|
|
264
|
+
name, by = breakdown.metric, breakdown.by
|
|
265
|
+
metric = semantic.metrics[name]
|
|
266
|
+
inputs = semantic.simple_inputs(name)
|
|
267
|
+
if metric.kind in ("simple", "count"):
|
|
268
|
+
exact = _exact(metric)
|
|
269
|
+
members = [
|
|
270
|
+
Member(member, _rounded(value, exact)) for member, value in raw[(name, by)].items()
|
|
271
|
+
]
|
|
272
|
+
else:
|
|
273
|
+
names = list(dict.fromkeys(member for dep in inputs for member in raw[(dep, by)]))
|
|
274
|
+
names.sort(key=lambda member: (member is None, member))
|
|
275
|
+
members = []
|
|
276
|
+
for member in names:
|
|
277
|
+
picked = {
|
|
278
|
+
dep: (
|
|
279
|
+
raw[(dep, by)][member]
|
|
280
|
+
if member in raw[(dep, by)]
|
|
281
|
+
else _empty(semantic.metrics[dep])
|
|
282
|
+
)
|
|
283
|
+
for dep in inputs
|
|
284
|
+
}
|
|
285
|
+
with localcontext(Context(prec=40)):
|
|
286
|
+
members.append(Member(member, _rounded(_evaluate(semantic, name, picked), False)))
|
|
287
|
+
members.sort(key=_sort_key)
|
|
288
|
+
breakdown.members = members[:limit]
|
|
289
|
+
rest = members[limit:]
|
|
290
|
+
breakdown.hidden = len(rest)
|
|
291
|
+
if not rest:
|
|
292
|
+
return
|
|
293
|
+
if _additive(metric):
|
|
294
|
+
total = sum((item.value for item in rest if item.value is not None), Decimal(0))
|
|
295
|
+
breakdown.other = _rounded(total, _exact(metric))
|
|
296
|
+
else:
|
|
297
|
+
breakdown.note = f"{len(rest)} smaller members left out: {name} cannot be added up"
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _period(start: dt.date, grain: str) -> str:
|
|
301
|
+
"""How a period is written: `2025-01-06` for a day or the Monday of a week, `2025-01`,
|
|
302
|
+
`2025-Q1`, `2025`. They sort in time order within a grain."""
|
|
303
|
+
if grain in ("day", "week"):
|
|
304
|
+
return start.isoformat()
|
|
305
|
+
if grain == "month":
|
|
306
|
+
return start.strftime("%Y-%m")
|
|
307
|
+
if grain == "quarter":
|
|
308
|
+
return f"{start.year}-Q{(start.month + 2) // 3}"
|
|
309
|
+
return str(start.year)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def by_period(
|
|
313
|
+
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], grain: str
|
|
314
|
+
) -> dict[str, Optional[dict[str, Optional[Number]]]]:
|
|
315
|
+
"""Each metric's value per `grain` of its time column, None for one with no time.
|
|
316
|
+
|
|
317
|
+
As `by_month` in `known_values` at any other grain: a period appears when a row of
|
|
318
|
+
the metric's table has a time in it, and a ratio or derived metric has periods when
|
|
319
|
+
every metric it is computed from has a time.
|
|
320
|
+
"""
|
|
321
|
+
if grain not in GRAINS:
|
|
322
|
+
raise ValueError(f"{grain!r} is not one of {', '.join(GRAINS)}")
|
|
323
|
+
computed = [
|
|
324
|
+
name for name, metric in semantic.metrics.items() if metric.kind in ("simple", "count")
|
|
325
|
+
]
|
|
326
|
+
read: dict[str, set[str]] = {}
|
|
327
|
+
for name in computed:
|
|
328
|
+
for table, columns in columns_read(semantic, name).items():
|
|
329
|
+
read.setdefault(table, set()).update(columns)
|
|
330
|
+
raw: dict[str, tuple[Any, Optional[dict[str, Any]]]] = {}
|
|
331
|
+
with _tables(semantic, frames, read) as (connection, relation):
|
|
332
|
+
for name in computed:
|
|
333
|
+
periods = None
|
|
334
|
+
if semantic.metrics[name].time is not None:
|
|
335
|
+
sql = query(semantic, name, relation, by_month=True, grain=grain)
|
|
336
|
+
periods = {
|
|
337
|
+
_period(start, grain): value
|
|
338
|
+
for start, value in connection.execute(sql).fetchall()
|
|
339
|
+
if start is not None
|
|
340
|
+
}
|
|
341
|
+
raw[name] = (None, periods)
|
|
342
|
+
values = _assemble(semantic, raw, seed=None, as_of=None).values
|
|
343
|
+
return {name: known.by_month for name, known in values.items()}
|
|
@@ -200,7 +200,11 @@ class _From:
|
|
|
200
200
|
|
|
201
201
|
|
|
202
202
|
def _parts(
|
|
203
|
-
semantic: SemanticModel,
|
|
203
|
+
semantic: SemanticModel,
|
|
204
|
+
metric: ResolvedMetric,
|
|
205
|
+
relation: Relation,
|
|
206
|
+
by_month: bool,
|
|
207
|
+
by: str | None = None,
|
|
204
208
|
) -> tuple[str, _From]:
|
|
205
209
|
# The total joins what its filter reads; by month, the time column's table as well.
|
|
206
210
|
needed = (
|
|
@@ -211,6 +215,8 @@ def _parts(
|
|
|
211
215
|
if by_month:
|
|
212
216
|
assert metric.time is not None
|
|
213
217
|
needed.add(metric.time.rpartition(".")[0])
|
|
218
|
+
if by is not None:
|
|
219
|
+
needed.add(by.rpartition(".")[0])
|
|
214
220
|
clauses = _From(metric, relation, needed)
|
|
215
221
|
|
|
216
222
|
def column_of(path: str) -> tuple[str, ColumnKind]:
|
|
@@ -229,37 +235,60 @@ def _parts(
|
|
|
229
235
|
|
|
230
236
|
|
|
231
237
|
def _query(
|
|
232
|
-
semantic: SemanticModel,
|
|
238
|
+
semantic: SemanticModel,
|
|
239
|
+
name: str,
|
|
240
|
+
relation: Relation,
|
|
241
|
+
by_month: bool,
|
|
242
|
+
by: str | None = None,
|
|
243
|
+
grain: str = "month",
|
|
233
244
|
) -> tuple[str, _From]:
|
|
234
245
|
metric = semantic.metrics[name]
|
|
235
|
-
aggregate, clauses = _parts(semantic, metric, relation, by_month)
|
|
246
|
+
aggregate, clauses = _parts(semantic, metric, relation, by_month, by)
|
|
236
247
|
joins = "\n".join(clauses.lines)
|
|
248
|
+
if by is not None:
|
|
249
|
+
table, column = by.rpartition(".")[0], by.rpartition(".")[2]
|
|
250
|
+
member = clauses.column(table, column)
|
|
251
|
+
return f"select {member} as member, {aggregate} as value\n{joins}\ngroup by 1", clauses
|
|
237
252
|
if not by_month:
|
|
238
253
|
return f"select {aggregate} as value\n{joins}", clauses
|
|
239
254
|
assert metric.time is not None
|
|
240
255
|
table, column = metric.time.rpartition(".")[0], metric.time.rpartition(".")[2]
|
|
241
|
-
|
|
256
|
+
instant = f"cast({clauses.column(table, column)} as timestamp)"
|
|
257
|
+
month = f"cast(date_trunc('{grain}', {instant}) as date)"
|
|
242
258
|
return (
|
|
243
259
|
f"select {month} as month, {aggregate} as value\n{joins}\ngroup by 1\norder by 1",
|
|
244
260
|
clauses,
|
|
245
261
|
)
|
|
246
262
|
|
|
247
263
|
|
|
248
|
-
def query(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
264
|
+
def query(
|
|
265
|
+
semantic: SemanticModel,
|
|
266
|
+
name: str,
|
|
267
|
+
relation: Relation,
|
|
268
|
+
*,
|
|
269
|
+
by_month: bool = False,
|
|
270
|
+
by: str | None = None,
|
|
271
|
+
grain: str = "month",
|
|
272
|
+
) -> str:
|
|
273
|
+
"""A simple metric or row count as a query: one row with its `value`; `by_month`, a
|
|
274
|
+
row per calendar month of its time column (`month` the month's first day), or per
|
|
275
|
+
`grain` of it (`day`, `week`, `quarter`, `year`; still in a column called `month`); or
|
|
276
|
+
`by` a column path in the metric's `joins`, a row per `member` of that column."""
|
|
277
|
+
return _query(semantic, name, relation, by_month, by, grain)[0]
|
|
252
278
|
|
|
253
279
|
|
|
254
|
-
def columns_read(semantic: SemanticModel, name: str) -> dict[str, set[str]]:
|
|
280
|
+
def columns_read(semantic: SemanticModel, name: str, by: str | None = None) -> dict[str, set[str]]:
|
|
255
281
|
"""Every column `query` reads for a simple metric or row count, by table key, its
|
|
256
282
|
`by_month` query's time column included: the columns its aggregate, filter, time and
|
|
257
283
|
joins name. Every table the query reads is a key, the metric's own table always, even
|
|
258
284
|
when no column of it is named (a row count with no filter: `count(*)`).
|
|
259
285
|
|
|
260
286
|
Recorded while the query is written rather than found in its text, so it cannot
|
|
261
|
-
miss a column however the column is quoted or aliased.
|
|
287
|
+
miss a column however the column is quoted or aliased. With `by`, those of its query
|
|
288
|
+
per member of that column instead.
|
|
262
289
|
"""
|
|
290
|
+
if by is not None:
|
|
291
|
+
return _query(semantic, name, str, by_month=False, by=by)[1].read
|
|
263
292
|
metric = semantic.metrics[name]
|
|
264
293
|
return _query(semantic, name, str, by_month=metric.time is not None)[1].read
|
|
265
294
|
|
|
@@ -38,7 +38,8 @@ from __future__ import annotations
|
|
|
38
38
|
|
|
39
39
|
import io
|
|
40
40
|
import json
|
|
41
|
-
from collections.abc import Mapping
|
|
41
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
42
|
+
from contextlib import contextmanager
|
|
42
43
|
from dataclasses import dataclass, field
|
|
43
44
|
from decimal import ROUND_HALF_EVEN, Context, Decimal, localcontext
|
|
44
45
|
from typing import Any, Optional, Union
|
|
@@ -186,29 +187,19 @@ def _kept(frame: pd.DataFrame, read: set[str]) -> list[Any]:
|
|
|
186
187
|
return kept or list(frame.columns[:1])
|
|
187
188
|
|
|
188
189
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
frames: Mapping[str, pd.DataFrame],
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
) -> KnownValues:
|
|
196
|
-
"""Every metric of `semantic` over `frames` (by table key: what the seeds hold)."""
|
|
190
|
+
@contextmanager
|
|
191
|
+
def _tables(
|
|
192
|
+
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], read: Mapping[str, set[str]]
|
|
193
|
+
) -> Iterator[tuple[Any, Callable[[str], str]]]:
|
|
194
|
+
"""A DuckDB connection holding the `read` columns of `frames` as their seeds read,
|
|
195
|
+
and the relation name of each table key."""
|
|
197
196
|
import duckdb
|
|
198
197
|
|
|
199
|
-
computed = [
|
|
200
|
-
name for name, metric in semantic.metrics.items() if metric.kind in ("simple", "count")
|
|
201
|
-
]
|
|
202
|
-
read: dict[str, set[str]] = {}
|
|
203
|
-
for name in computed:
|
|
204
|
-
for table, columns in columns_read(semantic, name).items():
|
|
205
|
-
read.setdefault(table, set()).update(columns)
|
|
206
198
|
keys = [table for table in semantic.entities if table in read]
|
|
207
199
|
for key in keys:
|
|
208
200
|
if key not in frames:
|
|
209
201
|
raise ValueError(f"no generated rows for table {key!r}, which a metric reads")
|
|
210
202
|
names = {key: f"m2d_{index}" for index, key in enumerate(keys)}
|
|
211
|
-
relation = names.__getitem__
|
|
212
203
|
|
|
213
204
|
connection = duckdb.connect(":memory:")
|
|
214
205
|
try:
|
|
@@ -226,7 +217,28 @@ def known_values(
|
|
|
226
217
|
)
|
|
227
218
|
connection.unregister(f"{names[key]}_frame")
|
|
228
219
|
del text
|
|
229
|
-
|
|
220
|
+
yield connection, names.__getitem__
|
|
221
|
+
finally:
|
|
222
|
+
connection.close()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def known_values(
|
|
226
|
+
semantic: SemanticModel,
|
|
227
|
+
frames: Mapping[str, pd.DataFrame],
|
|
228
|
+
*,
|
|
229
|
+
seed: Optional[int] = None,
|
|
230
|
+
as_of: Optional[str] = None,
|
|
231
|
+
) -> KnownValues:
|
|
232
|
+
"""Every metric of `semantic` over `frames` (by table key: what the seeds hold)."""
|
|
233
|
+
computed = [
|
|
234
|
+
name for name, metric in semantic.metrics.items() if metric.kind in ("simple", "count")
|
|
235
|
+
]
|
|
236
|
+
read: dict[str, set[str]] = {}
|
|
237
|
+
for name in computed:
|
|
238
|
+
for table, columns in columns_read(semantic, name).items():
|
|
239
|
+
read.setdefault(table, set()).update(columns)
|
|
240
|
+
raw: dict[str, tuple[Any, Optional[dict[str, Any]]]] = {}
|
|
241
|
+
with _tables(semantic, frames, read) as (connection, relation):
|
|
230
242
|
for name in computed:
|
|
231
243
|
total = connection.execute(query(semantic, name, relation)).fetchone()
|
|
232
244
|
by_month = None
|
|
@@ -236,8 +248,6 @@ def known_values(
|
|
|
236
248
|
month.strftime("%Y-%m"): value for month, value in rows if month is not None
|
|
237
249
|
}
|
|
238
250
|
raw[name] = (total[0] if total else None, by_month)
|
|
239
|
-
finally:
|
|
240
|
-
connection.close()
|
|
241
251
|
return _assemble(semantic, raw, seed=seed, as_of=as_of)
|
|
242
252
|
|
|
243
253
|
|
|
@@ -119,6 +119,7 @@ model2data/metrics/lightdash.py
|
|
|
119
119
|
model2data/metrics/ossie.py
|
|
120
120
|
model2data/metrics/reader.py
|
|
121
121
|
model2data/metrics/semantic.py
|
|
122
|
+
model2data/metrics/slices.py
|
|
122
123
|
model2data/metrics/sql.py
|
|
123
124
|
model2data/metrics/types.py
|
|
124
125
|
model2data/metrics/values.py
|
|
@@ -176,6 +177,7 @@ tests/test_metrics_lightdash.py
|
|
|
176
177
|
tests/test_metrics_ossie.py
|
|
177
178
|
tests/test_metrics_reader.py
|
|
178
179
|
tests/test_metrics_semantic.py
|
|
180
|
+
tests/test_metrics_slices.py
|
|
179
181
|
tests/test_metrics_spec.py
|
|
180
182
|
tests/test_metrics_values.py
|
|
181
183
|
tests/test_model.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "model2data"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.18.0"
|
|
8
8
|
description = "Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values"
|
|
9
9
|
readme = "README_PYPI.md"
|
|
10
10
|
requires-python = ">=3.10"
|