model2data 1.18.0__tar.gz → 1.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.18.0/model2data.egg-info → model2data-1.19.0}/PKG-INFO +1 -1
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/__init__.py +16 -1
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/slices.py +24 -6
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/values.py +190 -11
- {model2data-1.18.0 → model2data-1.19.0/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.18.0 → model2data-1.19.0}/model2data.egg-info/SOURCES.txt +1 -0
- {model2data-1.18.0 → model2data-1.19.0}/pyproject.toml +1 -1
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_determinism.py +9 -0
- model2data-1.19.0/tests/test_metrics_prepared.py +350 -0
- {model2data-1.18.0 → model2data-1.19.0}/LICENSE +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/README.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/README_PYPI.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/LICENSE +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/VENDORED.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/base.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/column.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/index.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/project.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/table.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/constants.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/database.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/column.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/common.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/index.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/project.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/records.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/exceptions.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/parser.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/base.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/_vendor/pydbml/tools.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/cli.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/hint_tests.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/naming.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/project.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_when_tests.sql +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/history_macros/model2data_history_tests.sql +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/dbt/tests.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/defects/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/defects/apply.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/defects/checks.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/defects/presets.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/defects/report.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/core.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/days.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/faker.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/hints.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/history.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/kinds.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/options.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/parents.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/relationships.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/timeline.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/generate/when.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/guide/setup.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/guide/triage.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/guide/tune.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/check.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/columns.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/dbt.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/expression.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/graph.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/infer.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/lightdash.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/ossie.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/reader.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/semantic.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/sql.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/metrics/types.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/_yaml.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/dbml.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/document.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/dump.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/engine.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/errors.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/reader.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/types.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/model/validate.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/output.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/parse/__init__.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/parse/dbml.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/PUBLISHING.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/README.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/examples/coffee_webshop.metrics.yml +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/metrics/README.md +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/metrics/metrics.schema.json +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/spec/model.schema.json +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data/utils.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/setup.cfg +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_cli.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_cli_model.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_column_time_hints.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_days.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbml_parser.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbt_integration.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbt_naming.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbt_project.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_dbt_tests.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_defects.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_defects_dbt.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_defects_replay.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_defects_report.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_defects_review.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_distributions.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_examples_model.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_fast_paths.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_generation.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_guide.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_hint_tests.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_history.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_issue_lines.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_lone_country.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_cli.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_dbt.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_expression.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_lightdash.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_ossie.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_reader.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_semantic.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_slices.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_spec.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_metrics_values.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_model.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_model_dbml.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_model_edges.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_model_messages.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_model_vocabulary.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_options.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_parent_dates.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_release_stress.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_reproducibility.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_row_identity.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_shaping.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_spec_alignment.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_spec_schema.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_table_seeds.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_timeline.py +0 -0
- {model2data-1.18.0 → model2data-1.19.0}/tests/test_when.py +0 -0
|
@@ -14,6 +14,9 @@ generator never reads it: a model generates the same data with or without one.
|
|
|
14
14
|
>>> metrics.known_values(narrowed.semantic, frames) # the same, on those rows
|
|
15
15
|
>>> metrics.breakdowns(semantic, frames, [("revenue", "customers.country")])
|
|
16
16
|
>>> metrics.by_period(semantic, frames, "week") # any grain, ISO weeks
|
|
17
|
+
>>> with metrics.prepare(frames) as prepared: # many calls, frames read once
|
|
18
|
+
... metrics.known_values(semantic, prepared)
|
|
19
|
+
... metrics.by_period(semantic, prepared, "quarter")
|
|
17
20
|
>>> export = metrics.to_ossie(semantic) # Apache Ossie 0.1.1
|
|
18
21
|
>>> export.to_yaml(), export.lossiness
|
|
19
22
|
>>> metrics.to_lightdash(semantic).to_yaml() # Lightdash meta, dbt YAML
|
|
@@ -56,6 +59,8 @@ from model2data.metrics.slices import (
|
|
|
56
59
|
breakdowns,
|
|
57
60
|
by_period,
|
|
58
61
|
filtered,
|
|
62
|
+
is_additive,
|
|
63
|
+
period_label,
|
|
59
64
|
)
|
|
60
65
|
from model2data.metrics.types import (
|
|
61
66
|
AGGREGATIONS,
|
|
@@ -68,7 +73,13 @@ from model2data.metrics.types import (
|
|
|
68
73
|
Ratio,
|
|
69
74
|
Where,
|
|
70
75
|
)
|
|
71
|
-
from model2data.metrics.values import
|
|
76
|
+
from model2data.metrics.values import (
|
|
77
|
+
KnownValue,
|
|
78
|
+
KnownValues,
|
|
79
|
+
PreparedFrames,
|
|
80
|
+
known_values,
|
|
81
|
+
prepare,
|
|
82
|
+
)
|
|
72
83
|
|
|
73
84
|
__all__ = [
|
|
74
85
|
"AGGREGATIONS",
|
|
@@ -97,6 +108,7 @@ __all__ = [
|
|
|
97
108
|
"Metrics",
|
|
98
109
|
"MetricsError",
|
|
99
110
|
"OssieExport",
|
|
111
|
+
"PreparedFrames",
|
|
100
112
|
"Ratio",
|
|
101
113
|
"Relationship",
|
|
102
114
|
"ResolvedMetric",
|
|
@@ -106,6 +118,7 @@ __all__ = [
|
|
|
106
118
|
"check_document",
|
|
107
119
|
"from_dict",
|
|
108
120
|
"inferred_metrics",
|
|
121
|
+
"is_additive",
|
|
109
122
|
"is_metrics_file",
|
|
110
123
|
"breakdowns",
|
|
111
124
|
"by_period",
|
|
@@ -115,6 +128,8 @@ __all__ = [
|
|
|
115
128
|
"metric_test_sql",
|
|
116
129
|
"metrics_stem",
|
|
117
130
|
"parse_expression",
|
|
131
|
+
"period_label",
|
|
132
|
+
"prepare",
|
|
118
133
|
"resolve",
|
|
119
134
|
"sibling_model",
|
|
120
135
|
"to_lightdash",
|
|
@@ -200,10 +200,18 @@ def filtered(
|
|
|
200
200
|
return Filtered(replace(semantic, metrics=metrics), tuple(unfiltered))
|
|
201
201
|
|
|
202
202
|
|
|
203
|
-
def
|
|
203
|
+
def is_additive(metric: ResolvedMetric) -> bool:
|
|
204
|
+
"""Whether `metric`'s values for separate sets of rows add up to its value for all of
|
|
205
|
+
them: a row count, or a sum or count of a column. An average, a median, a minimum, a
|
|
206
|
+
maximum, a distinct count, a ratio or a derived metric does not, so its members or
|
|
207
|
+
periods must not be summed into a total, and `breakdowns` leaves out the members past
|
|
208
|
+
its cap rather than summing them into `other`."""
|
|
204
209
|
return metric.kind == "count" or metric.agg in ("sum", "count")
|
|
205
210
|
|
|
206
211
|
|
|
212
|
+
_additive = is_additive # the name it had before it was public
|
|
213
|
+
|
|
214
|
+
|
|
207
215
|
def _sort_key(member: Member) -> tuple:
|
|
208
216
|
value = member.value
|
|
209
217
|
return (value is None, -value if value is not None else 0, member.member is None, member.member)
|
|
@@ -290,16 +298,23 @@ def _fill(
|
|
|
290
298
|
breakdown.hidden = len(rest)
|
|
291
299
|
if not rest:
|
|
292
300
|
return
|
|
293
|
-
if
|
|
301
|
+
if is_additive(metric):
|
|
294
302
|
total = sum((item.value for item in rest if item.value is not None), Decimal(0))
|
|
295
303
|
breakdown.other = _rounded(total, _exact(metric))
|
|
296
304
|
else:
|
|
297
305
|
breakdown.note = f"{len(rest)} smaller members left out: {name} cannot be added up"
|
|
298
306
|
|
|
299
307
|
|
|
300
|
-
def
|
|
301
|
-
"""
|
|
302
|
-
`2025-
|
|
308
|
+
def period_label(start: dt.date, grain: str) -> str:
|
|
309
|
+
"""The label `by_period` gives the `grain` period starting on `start`: `2025-01-06` for a
|
|
310
|
+
day or the Monday of an ISO week, `2025-01` for a month, `2025-Q1` for a quarter, `2025`
|
|
311
|
+
for a year. Labels sort in time order within a grain.
|
|
312
|
+
|
|
313
|
+
`start` may be any day of the period for a month, a quarter or a year; for a week it
|
|
314
|
+
must be the Monday, since a week is labelled by the day it starts.
|
|
315
|
+
"""
|
|
316
|
+
if grain not in GRAINS:
|
|
317
|
+
raise ValueError(f"{grain!r} is not one of {', '.join(GRAINS)}")
|
|
303
318
|
if grain in ("day", "week"):
|
|
304
319
|
return start.isoformat()
|
|
305
320
|
if grain == "month":
|
|
@@ -309,6 +324,9 @@ def _period(start: dt.date, grain: str) -> str:
|
|
|
309
324
|
return str(start.year)
|
|
310
325
|
|
|
311
326
|
|
|
327
|
+
_period = period_label # the name it had before it was public
|
|
328
|
+
|
|
329
|
+
|
|
312
330
|
def by_period(
|
|
313
331
|
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], grain: str
|
|
314
332
|
) -> dict[str, Optional[dict[str, Optional[Number]]]]:
|
|
@@ -334,7 +352,7 @@ def by_period(
|
|
|
334
352
|
if semantic.metrics[name].time is not None:
|
|
335
353
|
sql = query(semantic, name, relation, by_month=True, grain=grain)
|
|
336
354
|
periods = {
|
|
337
|
-
|
|
355
|
+
period_label(start, grain): value
|
|
338
356
|
for start, value in connection.execute(sql).fetchall()
|
|
339
357
|
if start is not None
|
|
340
358
|
}
|
|
@@ -38,6 +38,8 @@ from __future__ import annotations
|
|
|
38
38
|
|
|
39
39
|
import io
|
|
40
40
|
import json
|
|
41
|
+
import threading
|
|
42
|
+
from collections import Counter
|
|
41
43
|
from collections.abc import Callable, Iterator, Mapping
|
|
42
44
|
from contextlib import contextmanager
|
|
43
45
|
from dataclasses import dataclass, field
|
|
@@ -187,35 +189,212 @@ def _kept(frame: pd.DataFrame, read: set[str]) -> list[Any]:
|
|
|
187
189
|
return kept or list(frame.columns[:1])
|
|
188
190
|
|
|
189
191
|
|
|
192
|
+
def _load(connection: Any, name: str, text: pd.DataFrame, row: Optional[str] = None) -> None:
|
|
193
|
+
"""Create table `name` from text columns, each null where dbt's seed loader reads one.
|
|
194
|
+
|
|
195
|
+
With `row`, the table also has that column: each row's position in the frame, 0 first,
|
|
196
|
+
which is the order the rows are stored in.
|
|
197
|
+
"""
|
|
198
|
+
columns = [
|
|
199
|
+
f"{_NULL_OR_TEXT.format(value=f'cast({quote(c)} as varchar)')} as {quote(c)}"
|
|
200
|
+
for c in text.columns
|
|
201
|
+
]
|
|
202
|
+
if row is not None:
|
|
203
|
+
text = text.assign(**{row: range(len(text))})
|
|
204
|
+
columns.insert(0, quote(row))
|
|
205
|
+
connection.register(f"{name}_frame", text)
|
|
206
|
+
order = f" order by {quote(row)}" if row is not None else ""
|
|
207
|
+
connection.execute(
|
|
208
|
+
f"create table {name} as select {', '.join(columns)} from {name}_frame{order}"
|
|
209
|
+
)
|
|
210
|
+
connection.unregister(f"{name}_frame")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _row_column(frame: pd.DataFrame) -> str:
|
|
214
|
+
"""A name for the row-number column that no column of `frame` has, compared the way
|
|
215
|
+
`_kept` compares names, so no query can name it."""
|
|
216
|
+
taken = {str(column).casefold() for column in frame.columns}
|
|
217
|
+
name = "__m2d_row"
|
|
218
|
+
while name.casefold() in taken:
|
|
219
|
+
name = f"_{name}"
|
|
220
|
+
return name
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class PreparedFrames(Mapping[str, pd.DataFrame]):
|
|
224
|
+
"""A run's frames, read the way their seeds are once, for asking many questions of.
|
|
225
|
+
|
|
226
|
+
Every function of `model2data.metrics` that computes over frames takes these in
|
|
227
|
+
place of the plain mapping and gives the same values, byte for byte. What it saves is
|
|
228
|
+
the reading. Each column a query needs is turned into its seed's text the first time
|
|
229
|
+
some call needs it, and loaded into one in-memory DuckDB database that every later call
|
|
230
|
+
queries, so no column is converted or loaded twice. Use it when the same frames answer
|
|
231
|
+
more than one call (`known_values`, `by_period` at several grains, `breakdowns`, a
|
|
232
|
+
`filtered` view each); a single call gains nothing.
|
|
233
|
+
|
|
234
|
+
It is a read-only mapping of the frames it was given. The frames must not change
|
|
235
|
+
while it is in use: what it loaded would no longer be what they hold.
|
|
236
|
+
|
|
237
|
+
Calls from several threads at once are safe. Converting and loading happen under a
|
|
238
|
+
lock, so a column is converted once however many threads ask for it; each call then
|
|
239
|
+
queries through a DuckDB cursor of its own, outside the lock.
|
|
240
|
+
|
|
241
|
+
The database lives until `close()`, the end of a `with` block, or the object being
|
|
242
|
+
garbage collected. Close it while no call is running; a call after that raises
|
|
243
|
+
`ValueError`.
|
|
244
|
+
|
|
245
|
+
>>> with metrics.prepare(frames) as prepared:
|
|
246
|
+
... values = metrics.known_values(semantic, prepared)
|
|
247
|
+
... quarters = metrics.by_period(semantic, prepared, "quarter")
|
|
248
|
+
"""
|
|
249
|
+
|
|
250
|
+
def __init__(self, frames: Mapping[str, pd.DataFrame]) -> None:
|
|
251
|
+
self._frames = dict(frames)
|
|
252
|
+
self._lock = threading.Lock()
|
|
253
|
+
self._connection: Any = None
|
|
254
|
+
self._closed = False
|
|
255
|
+
# Table key -> the database table holding its loaded columns, and those columns.
|
|
256
|
+
self._loaded: dict[str, tuple[str, list[Any]]] = {}
|
|
257
|
+
# Table key -> its tables' row-number column, which no view shows.
|
|
258
|
+
self._rows: dict[str, str] = {}
|
|
259
|
+
self._made = 0
|
|
260
|
+
# Tables a call is reading, and tables replaced by a wider one, dropped when unread.
|
|
261
|
+
self._reading: Counter[str] = Counter()
|
|
262
|
+
self._replaced: set[str] = set()
|
|
263
|
+
|
|
264
|
+
def __getitem__(self, key: str) -> pd.DataFrame:
|
|
265
|
+
return self._frames[key]
|
|
266
|
+
|
|
267
|
+
def __iter__(self) -> Iterator[str]:
|
|
268
|
+
return iter(self._frames)
|
|
269
|
+
|
|
270
|
+
def __len__(self) -> int:
|
|
271
|
+
return len(self._frames)
|
|
272
|
+
|
|
273
|
+
def __repr__(self) -> str:
|
|
274
|
+
state = "closed" if self._closed else f"{len(self._loaded)} tables loaded"
|
|
275
|
+
return f"<PreparedFrames {list(self._frames)!r}, {state}>"
|
|
276
|
+
|
|
277
|
+
def __enter__(self) -> PreparedFrames:
|
|
278
|
+
return self
|
|
279
|
+
|
|
280
|
+
def __exit__(self, *_: object) -> None:
|
|
281
|
+
self.close()
|
|
282
|
+
|
|
283
|
+
@property
|
|
284
|
+
def closed(self) -> bool:
|
|
285
|
+
return self._closed
|
|
286
|
+
|
|
287
|
+
def close(self) -> None:
|
|
288
|
+
"""Close the DuckDB database, and with it everything loaded. Closing twice is fine."""
|
|
289
|
+
with self._lock:
|
|
290
|
+
self._closed = True
|
|
291
|
+
self._loaded.clear()
|
|
292
|
+
if self._connection is not None:
|
|
293
|
+
self._connection.close()
|
|
294
|
+
self._connection = None
|
|
295
|
+
|
|
296
|
+
@contextmanager
|
|
297
|
+
def _relations(self, wanted: Mapping[str, tuple[str, list[Any]]]) -> Iterator[Any]:
|
|
298
|
+
"""A cursor in which each relation name of `wanted` is a view of its table key's
|
|
299
|
+
loaded table, which holds at least the columns asked for."""
|
|
300
|
+
import duckdb
|
|
301
|
+
|
|
302
|
+
with self._lock:
|
|
303
|
+
if self._closed:
|
|
304
|
+
raise ValueError("these prepared frames are closed")
|
|
305
|
+
if self._connection is None:
|
|
306
|
+
self._connection = duckdb.connect(":memory:")
|
|
307
|
+
self._connection.execute("set threads = 1")
|
|
308
|
+
tables = {
|
|
309
|
+
relation: self._table(key, columns) for relation, (key, columns) in wanted.items()
|
|
310
|
+
}
|
|
311
|
+
rows = {relation: self._rows[key] for relation, (key, _) in wanted.items()}
|
|
312
|
+
self._reading.update(tables.values())
|
|
313
|
+
cursor = self._connection.cursor()
|
|
314
|
+
try:
|
|
315
|
+
for relation, table in tables.items():
|
|
316
|
+
# Temporary, so the view is this cursor's alone; it shows the frame's
|
|
317
|
+
# columns only, never the row number.
|
|
318
|
+
cursor.execute(
|
|
319
|
+
f"create temp view {relation} as "
|
|
320
|
+
f"select * exclude ({quote(rows[relation])}) from {table}"
|
|
321
|
+
)
|
|
322
|
+
yield cursor
|
|
323
|
+
finally:
|
|
324
|
+
cursor.close()
|
|
325
|
+
with self._lock:
|
|
326
|
+
self._reading.subtract(tables.values())
|
|
327
|
+
unread = {table for table in self._replaced if self._reading[table] <= 0}
|
|
328
|
+
self._replaced -= unread
|
|
329
|
+
if self._connection is not None:
|
|
330
|
+
for table in sorted(unread):
|
|
331
|
+
self._connection.execute(f"drop table {table}")
|
|
332
|
+
|
|
333
|
+
def _table(self, key: str, columns: list[Any]) -> str:
|
|
334
|
+
"""The table holding `columns` of `key`, converting and loading the ones that are
|
|
335
|
+
not loaded yet. Called under the lock."""
|
|
336
|
+
table, loaded = self._loaded.get(key, ("", []))
|
|
337
|
+
missing = [column for column in columns if column not in loaded]
|
|
338
|
+
if not missing:
|
|
339
|
+
return table
|
|
340
|
+
self._made += 1
|
|
341
|
+
wider = f"m2d_prepared_{self._made}"
|
|
342
|
+
frame = self._frames[key]
|
|
343
|
+
row = self._rows.setdefault(key, _row_column(frame))
|
|
344
|
+
text = _text_columns(frame, missing)
|
|
345
|
+
if not table:
|
|
346
|
+
_load(self._connection, wider, text, row)
|
|
347
|
+
else:
|
|
348
|
+
# Only the new columns are converted; the loaded ones are kept, matched to the
|
|
349
|
+
# new ones by row number and stored in the frame's order, as the first load was.
|
|
350
|
+
_load(self._connection, f"{wider}_new", text, row)
|
|
351
|
+
self._connection.execute(
|
|
352
|
+
f"create table {wider} as select * from {table} "
|
|
353
|
+
f"join {wider}_new using ({quote(row)}) order by {quote(row)}"
|
|
354
|
+
)
|
|
355
|
+
self._connection.execute(f"drop table {wider}_new")
|
|
356
|
+
self._replaced.add(table)
|
|
357
|
+
self._loaded[key] = (wider, [*loaded, *missing])
|
|
358
|
+
return wider
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def prepare(frames: Mapping[str, pd.DataFrame]) -> PreparedFrames:
|
|
362
|
+
"""`frames` (by table key, what the seeds hold) prepared for many metric calls; frames
|
|
363
|
+
already prepared are returned as they are. See `PreparedFrames`."""
|
|
364
|
+
return frames if isinstance(frames, PreparedFrames) else PreparedFrames(frames)
|
|
365
|
+
|
|
366
|
+
|
|
190
367
|
@contextmanager
|
|
191
368
|
def _tables(
|
|
192
369
|
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], read: Mapping[str, set[str]]
|
|
193
370
|
) -> Iterator[tuple[Any, Callable[[str], str]]]:
|
|
194
371
|
"""A DuckDB connection holding the `read` columns of `frames` as their seeds read,
|
|
195
|
-
and the relation name of each table key.
|
|
196
|
-
import duckdb
|
|
372
|
+
and the relation name of each table key.
|
|
197
373
|
|
|
374
|
+
Plain frames are read for this call alone, into a database of its own. Prepared frames
|
|
375
|
+
give a cursor on their shared database, with the same relation names.
|
|
376
|
+
"""
|
|
198
377
|
keys = [table for table in semantic.entities if table in read]
|
|
199
378
|
for key in keys:
|
|
200
379
|
if key not in frames:
|
|
201
380
|
raise ValueError(f"no generated rows for table {key!r}, which a metric reads")
|
|
202
381
|
names = {key: f"m2d_{index}" for index, key in enumerate(keys)}
|
|
203
382
|
|
|
383
|
+
if isinstance(frames, PreparedFrames):
|
|
384
|
+
wanted = {names[key]: (key, _kept(frames[key], read[key])) for key in keys}
|
|
385
|
+
with frames._relations(wanted) as cursor:
|
|
386
|
+
yield cursor, names.__getitem__
|
|
387
|
+
return
|
|
388
|
+
|
|
389
|
+
import duckdb
|
|
390
|
+
|
|
204
391
|
connection = duckdb.connect(":memory:")
|
|
205
392
|
try:
|
|
206
393
|
connection.execute("set threads = 1")
|
|
207
394
|
for key in keys:
|
|
208
395
|
# One table's text at a time: it is freed once DuckDB holds its own copy.
|
|
209
396
|
text = _text_columns(frames[key], _kept(frames[key], read[key]))
|
|
210
|
-
connection
|
|
211
|
-
columns = ", ".join(
|
|
212
|
-
f"{_NULL_OR_TEXT.format(value=f'cast({quote(c)} as varchar)')} as {quote(c)}"
|
|
213
|
-
for c in text.columns
|
|
214
|
-
)
|
|
215
|
-
connection.execute(
|
|
216
|
-
f"create table {names[key]} as select {columns} from {names[key]}_frame"
|
|
217
|
-
)
|
|
218
|
-
connection.unregister(f"{names[key]}_frame")
|
|
397
|
+
_load(connection, names[key], text)
|
|
219
398
|
del text
|
|
220
399
|
yield connection, names.__getitem__
|
|
221
400
|
finally:
|
|
@@ -175,6 +175,7 @@ tests/test_metrics_dbt.py
|
|
|
175
175
|
tests/test_metrics_expression.py
|
|
176
176
|
tests/test_metrics_lightdash.py
|
|
177
177
|
tests/test_metrics_ossie.py
|
|
178
|
+
tests/test_metrics_prepared.py
|
|
178
179
|
tests/test_metrics_reader.py
|
|
179
180
|
tests/test_metrics_semantic.py
|
|
180
181
|
tests/test_metrics_slices.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "model2data"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.19.0"
|
|
8
8
|
description = "Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values"
|
|
9
9
|
readme = "README_PYPI.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -78,6 +78,15 @@ _case(
|
|
|
78
78
|
"--days",
|
|
79
79
|
"2",
|
|
80
80
|
)
|
|
81
|
+
# The e-commerce example with its own metrics file and the Lightdash meta: recorded on 1.18.0,
|
|
82
|
+
# before metric calls could share prepared frames, which must not change a byte of it.
|
|
83
|
+
_case(
|
|
84
|
+
"metrics-ecommerce",
|
|
85
|
+
_ECOM,
|
|
86
|
+
"--metrics",
|
|
87
|
+
str(EXAMPLES / "ecommerce.metrics.yml"),
|
|
88
|
+
"--lightdash",
|
|
89
|
+
)
|
|
81
90
|
for _locale in ("en_US", "nl_BE", "de_DE", "fr_FR", "ja_JP"):
|
|
82
91
|
_case(f"locale-{_locale}", _ECOM, "--locale", _locale)
|
|
83
92
|
|