model2data 1.17.0__tar.gz → 1.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-1.17.0/model2data.egg-info → model2data-1.19.0}/PKG-INFO +1 -1
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/__init__.py +16 -1
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/slices.py +60 -9
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/values.py +190 -11
- {model2data-1.17.0 → model2data-1.19.0/model2data.egg-info}/PKG-INFO +1 -1
- {model2data-1.17.0 → model2data-1.19.0}/model2data.egg-info/SOURCES.txt +1 -0
- {model2data-1.17.0 → model2data-1.19.0}/pyproject.toml +1 -1
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_determinism.py +9 -0
- model2data-1.19.0/tests/test_metrics_prepared.py +350 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_slices.py +67 -0
- {model2data-1.17.0 → model2data-1.19.0}/LICENSE +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/README.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/README_PYPI.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/LICENSE +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/VENDORED.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/base.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/column.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/enum.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/expression.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/index.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/project.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/reference.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/sticky_note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/table.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/_classes/table_group.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/classes/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/constants.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/database.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/column.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/common.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/enum.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/generic.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/index.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/project.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/records.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/reference.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/sticky_note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table_group.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/definitions/table_partial.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/exceptions.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/blueprints.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/parser/parser.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/base.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/column.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/enum.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/expression.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/index.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/project.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/reference.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/renderer.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/sticky_note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/table.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/table_group.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/dbml/default/utils.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/column.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/enum.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/expression.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/index.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/note.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/reference.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/renderer.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/table.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/renderer/sql/default/utils.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/_vendor/pydbml/tools.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/cli.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/hint_tests.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/naming.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/project.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/dbt_project.yml.jinja +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_hint_tests.sql +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/hint_macros/model2data_when_tests.sql +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/history_macros/model2data_history_tests.sql +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/macros/generate_schema_name.sql +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/templates/profiles.yml.jinja +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/dbt/tests.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/defects/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/defects/apply.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/defects/checks.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/defects/presets.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/defects/report.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/core.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/days.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/faker.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/hints.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/history.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/kinds.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/options.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/parents.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/relationships.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/timeline.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/generate/when.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/guide/setup.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/guide/triage.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/guide/tune.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/check.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/columns.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/dbt.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/expression.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/graph.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/infer.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/lightdash.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/ossie.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/reader.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/semantic.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/sql.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/metrics/types.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/_yaml.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/dbml.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/document.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/dump.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/engine.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/errors.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/reader.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/types.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/model/validate.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/output.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/parse/__init__.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/parse/dbml.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/PUBLISHING.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/README.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/examples/coffee_webshop.metrics.yml +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/examples/coffee_webshop.model2data.yml +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/metrics/README.md +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/metrics/metrics.schema.json +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/spec/model.schema.json +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data/utils.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data.egg-info/requires.txt +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/model2data.egg-info/top_level.txt +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/setup.cfg +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_as_of_anchor.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_cli.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_cli_model.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_column_time_hints.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_coverage_gaps.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_days.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbml_parser.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbml_parser_fuzz.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbt_integration.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbt_naming.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbt_project.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_dbt_tests.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_defects.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_defects_dbt.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_defects_replay.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_defects_report.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_defects_review.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_distributions.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_examples_model.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_faker_name_inference.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_fast_paths.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_generation.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_guide.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_hint_tests.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_history.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_issue_lines.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_lone_country.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_cli.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_dbt.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_expression.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_lightdash.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_ossie.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_reader.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_semantic.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_spec.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_metrics_values.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_model.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_model_dbml.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_model_edges.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_model_messages.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_model_vocabulary.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_options.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_parent_dates.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_release_stress.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_reproducibility.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_row_identity.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_shaping.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_spec_alignment.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_spec_schema.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_table_seeds.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_timeline.py +0 -0
- {model2data-1.17.0 → model2data-1.19.0}/tests/test_when.py +0 -0
|
@@ -14,6 +14,9 @@ generator never reads it: a model generates the same data with or without one.
|
|
|
14
14
|
>>> metrics.known_values(narrowed.semantic, frames) # the same, on those rows
|
|
15
15
|
>>> metrics.breakdowns(semantic, frames, [("revenue", "customers.country")])
|
|
16
16
|
>>> metrics.by_period(semantic, frames, "week") # any grain, ISO weeks
|
|
17
|
+
>>> with metrics.prepare(frames) as prepared: # many calls, frames read once
|
|
18
|
+
... metrics.known_values(semantic, prepared)
|
|
19
|
+
... metrics.by_period(semantic, prepared, "quarter")
|
|
17
20
|
>>> export = metrics.to_ossie(semantic) # Apache Ossie 0.1.1
|
|
18
21
|
>>> export.to_yaml(), export.lossiness
|
|
19
22
|
>>> metrics.to_lightdash(semantic).to_yaml() # Lightdash meta, dbt YAML
|
|
@@ -56,6 +59,8 @@ from model2data.metrics.slices import (
|
|
|
56
59
|
breakdowns,
|
|
57
60
|
by_period,
|
|
58
61
|
filtered,
|
|
62
|
+
is_additive,
|
|
63
|
+
period_label,
|
|
59
64
|
)
|
|
60
65
|
from model2data.metrics.types import (
|
|
61
66
|
AGGREGATIONS,
|
|
@@ -68,7 +73,13 @@ from model2data.metrics.types import (
|
|
|
68
73
|
Ratio,
|
|
69
74
|
Where,
|
|
70
75
|
)
|
|
71
|
-
from model2data.metrics.values import
|
|
76
|
+
from model2data.metrics.values import (
|
|
77
|
+
KnownValue,
|
|
78
|
+
KnownValues,
|
|
79
|
+
PreparedFrames,
|
|
80
|
+
known_values,
|
|
81
|
+
prepare,
|
|
82
|
+
)
|
|
72
83
|
|
|
73
84
|
__all__ = [
|
|
74
85
|
"AGGREGATIONS",
|
|
@@ -97,6 +108,7 @@ __all__ = [
|
|
|
97
108
|
"Metrics",
|
|
98
109
|
"MetricsError",
|
|
99
110
|
"OssieExport",
|
|
111
|
+
"PreparedFrames",
|
|
100
112
|
"Ratio",
|
|
101
113
|
"Relationship",
|
|
102
114
|
"ResolvedMetric",
|
|
@@ -106,6 +118,7 @@ __all__ = [
|
|
|
106
118
|
"check_document",
|
|
107
119
|
"from_dict",
|
|
108
120
|
"inferred_metrics",
|
|
121
|
+
"is_additive",
|
|
109
122
|
"is_metrics_file",
|
|
110
123
|
"breakdowns",
|
|
111
124
|
"by_period",
|
|
@@ -115,6 +128,8 @@ __all__ = [
|
|
|
115
128
|
"metric_test_sql",
|
|
116
129
|
"metrics_stem",
|
|
117
130
|
"parse_expression",
|
|
131
|
+
"period_label",
|
|
132
|
+
"prepare",
|
|
118
133
|
"resolve",
|
|
119
134
|
"sibling_model",
|
|
120
135
|
"to_lightdash",
|
|
@@ -135,17 +135,50 @@ def _keeping(path: str, members: Sequence[Any], kind: ColumnKind) -> Where:
|
|
|
135
135
|
return Where(terms=alternatives, any=True)
|
|
136
136
|
|
|
137
137
|
|
|
138
|
-
def
|
|
138
|
+
def _within(path: str, start: Any, end: Any, kind: ColumnKind) -> Where:
|
|
139
|
+
"""The filter for rows whose date or timestamp `path` falls from `start` to `end`.
|
|
140
|
+
|
|
141
|
+
Both ends are included and either may be None. A timestamp's `end` given as a day
|
|
142
|
+
means all of that day, which is what a date picker's last day means.
|
|
143
|
+
"""
|
|
144
|
+
if kind not in ("date", "timestamp"):
|
|
145
|
+
raise ValueError(f"{path} is not a date or a timestamp ({kind})")
|
|
146
|
+
operators: dict[str, Any] = {}
|
|
147
|
+
if start is not None:
|
|
148
|
+
operators["gte"] = _value(start, kind, path)
|
|
149
|
+
if end is not None:
|
|
150
|
+
written = str(end)
|
|
151
|
+
if kind == "timestamp" and len(written) == 10:
|
|
152
|
+
day = dt.date.fromisoformat(_value(written, "date", path))
|
|
153
|
+
operators["lt"] = (day + dt.timedelta(days=1)).isoformat()
|
|
154
|
+
else:
|
|
155
|
+
operators["lte"] = _value(end, kind, path)
|
|
156
|
+
return Where(terms=[Where(terms=[Condition(path, operators)])])
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def filtered(
|
|
160
|
+
semantic: SemanticModel,
|
|
161
|
+
members: Mapping[str, Sequence[Any]],
|
|
162
|
+
*,
|
|
163
|
+
ranges: Optional[Mapping[str, tuple[Any, Any]]] = None,
|
|
164
|
+
) -> Filtered:
|
|
139
165
|
"""Every metric narrowed to rows whose dimension is one of its members.
|
|
140
166
|
|
|
141
|
-
`members` maps a column path to the values to keep, None for null.
|
|
142
|
-
|
|
167
|
+
`members` maps a column path to the values to keep, None for null. `ranges` maps a
|
|
168
|
+
date or timestamp column to its `(start, end)`, both included, None for open. All of
|
|
169
|
+
them together narrow as `and`. A dimension with no members keeps everything, and so
|
|
170
|
+
does a range open at both ends.
|
|
143
171
|
"""
|
|
144
172
|
wanted = {path: kept for path, kept in members.items() if len(kept) > 0}
|
|
145
173
|
terms = [
|
|
146
174
|
(path.rpartition(".")[0], _keeping(path, kept, _column(semantic, path)[1]))
|
|
147
175
|
for path, kept in wanted.items()
|
|
148
176
|
]
|
|
177
|
+
terms += [
|
|
178
|
+
(path.rpartition(".")[0], _within(path, start, end, _column(semantic, path)[1]))
|
|
179
|
+
for path, (start, end) in (ranges or {}).items()
|
|
180
|
+
if start is not None or end is not None
|
|
181
|
+
]
|
|
149
182
|
metrics = dict(semantic.metrics)
|
|
150
183
|
missed: set[str] = set()
|
|
151
184
|
for name, metric in semantic.metrics.items():
|
|
@@ -167,10 +200,18 @@ def filtered(semantic: SemanticModel, members: Mapping[str, Sequence[Any]]) -> F
|
|
|
167
200
|
return Filtered(replace(semantic, metrics=metrics), tuple(unfiltered))
|
|
168
201
|
|
|
169
202
|
|
|
170
|
-
def
|
|
203
|
+
def is_additive(metric: ResolvedMetric) -> bool:
|
|
204
|
+
"""Whether `metric`'s values for separate sets of rows add up to its value for all of
|
|
205
|
+
them: a row count, or a sum or count of a column. An average, a median, a minimum, a
|
|
206
|
+
maximum, a distinct count, a ratio or a derived metric does not, so its members or
|
|
207
|
+
periods must not be summed into a total, and `breakdowns` leaves out the members past
|
|
208
|
+
its cap rather than summing them into `other`."""
|
|
171
209
|
return metric.kind == "count" or metric.agg in ("sum", "count")
|
|
172
210
|
|
|
173
211
|
|
|
212
|
+
_additive = is_additive # the name it had before it was public
|
|
213
|
+
|
|
214
|
+
|
|
174
215
|
def _sort_key(member: Member) -> tuple:
|
|
175
216
|
value = member.value
|
|
176
217
|
return (value is None, -value if value is not None else 0, member.member is None, member.member)
|
|
@@ -257,16 +298,23 @@ def _fill(
|
|
|
257
298
|
breakdown.hidden = len(rest)
|
|
258
299
|
if not rest:
|
|
259
300
|
return
|
|
260
|
-
if
|
|
301
|
+
if is_additive(metric):
|
|
261
302
|
total = sum((item.value for item in rest if item.value is not None), Decimal(0))
|
|
262
303
|
breakdown.other = _rounded(total, _exact(metric))
|
|
263
304
|
else:
|
|
264
305
|
breakdown.note = f"{len(rest)} smaller members left out: {name} cannot be added up"
|
|
265
306
|
|
|
266
307
|
|
|
267
|
-
def
|
|
268
|
-
"""
|
|
269
|
-
`2025-
|
|
308
|
+
def period_label(start: dt.date, grain: str) -> str:
|
|
309
|
+
"""The label `by_period` gives the `grain` period starting on `start`: `2025-01-06` for a
|
|
310
|
+
day or the Monday of an ISO week, `2025-01` for a month, `2025-Q1` for a quarter, `2025`
|
|
311
|
+
for a year. Labels sort in time order within a grain.
|
|
312
|
+
|
|
313
|
+
`start` may be any day of the period for a month, a quarter or a year; for a week it
|
|
314
|
+
must be the Monday, since a week is labelled by the day it starts.
|
|
315
|
+
"""
|
|
316
|
+
if grain not in GRAINS:
|
|
317
|
+
raise ValueError(f"{grain!r} is not one of {', '.join(GRAINS)}")
|
|
270
318
|
if grain in ("day", "week"):
|
|
271
319
|
return start.isoformat()
|
|
272
320
|
if grain == "month":
|
|
@@ -276,6 +324,9 @@ def _period(start: dt.date, grain: str) -> str:
|
|
|
276
324
|
return str(start.year)
|
|
277
325
|
|
|
278
326
|
|
|
327
|
+
_period = period_label # the name it had before it was public
|
|
328
|
+
|
|
329
|
+
|
|
279
330
|
def by_period(
|
|
280
331
|
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], grain: str
|
|
281
332
|
) -> dict[str, Optional[dict[str, Optional[Number]]]]:
|
|
@@ -301,7 +352,7 @@ def by_period(
|
|
|
301
352
|
if semantic.metrics[name].time is not None:
|
|
302
353
|
sql = query(semantic, name, relation, by_month=True, grain=grain)
|
|
303
354
|
periods = {
|
|
304
|
-
|
|
355
|
+
period_label(start, grain): value
|
|
305
356
|
for start, value in connection.execute(sql).fetchall()
|
|
306
357
|
if start is not None
|
|
307
358
|
}
|
|
@@ -38,6 +38,8 @@ from __future__ import annotations
|
|
|
38
38
|
|
|
39
39
|
import io
|
|
40
40
|
import json
|
|
41
|
+
import threading
|
|
42
|
+
from collections import Counter
|
|
41
43
|
from collections.abc import Callable, Iterator, Mapping
|
|
42
44
|
from contextlib import contextmanager
|
|
43
45
|
from dataclasses import dataclass, field
|
|
@@ -187,35 +189,212 @@ def _kept(frame: pd.DataFrame, read: set[str]) -> list[Any]:
|
|
|
187
189
|
return kept or list(frame.columns[:1])
|
|
188
190
|
|
|
189
191
|
|
|
192
|
+
def _load(connection: Any, name: str, text: pd.DataFrame, row: Optional[str] = None) -> None:
|
|
193
|
+
"""Create table `name` from text columns, each null where dbt's seed loader reads one.
|
|
194
|
+
|
|
195
|
+
With `row`, the table also has that column: each row's position in the frame, 0 first,
|
|
196
|
+
which is the order the rows are stored in.
|
|
197
|
+
"""
|
|
198
|
+
columns = [
|
|
199
|
+
f"{_NULL_OR_TEXT.format(value=f'cast({quote(c)} as varchar)')} as {quote(c)}"
|
|
200
|
+
for c in text.columns
|
|
201
|
+
]
|
|
202
|
+
if row is not None:
|
|
203
|
+
text = text.assign(**{row: range(len(text))})
|
|
204
|
+
columns.insert(0, quote(row))
|
|
205
|
+
connection.register(f"{name}_frame", text)
|
|
206
|
+
order = f" order by {quote(row)}" if row is not None else ""
|
|
207
|
+
connection.execute(
|
|
208
|
+
f"create table {name} as select {', '.join(columns)} from {name}_frame{order}"
|
|
209
|
+
)
|
|
210
|
+
connection.unregister(f"{name}_frame")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _row_column(frame: pd.DataFrame) -> str:
|
|
214
|
+
"""A name for the row-number column that no column of `frame` has, compared the way
|
|
215
|
+
`_kept` compares names, so no query can name it."""
|
|
216
|
+
taken = {str(column).casefold() for column in frame.columns}
|
|
217
|
+
name = "__m2d_row"
|
|
218
|
+
while name.casefold() in taken:
|
|
219
|
+
name = f"_{name}"
|
|
220
|
+
return name
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class PreparedFrames(Mapping[str, pd.DataFrame]):
|
|
224
|
+
"""A run's frames, read the way their seeds are once, for asking many questions of.
|
|
225
|
+
|
|
226
|
+
Every function of `model2data.metrics` that computes over frames takes these in
|
|
227
|
+
place of the plain mapping and gives the same values, byte for byte. What it saves is
|
|
228
|
+
the reading. Each column a query needs is turned into its seed's text the first time
|
|
229
|
+
some call needs it, and loaded into one in-memory DuckDB database that every later call
|
|
230
|
+
queries, so no column is converted or loaded twice. Use it when the same frames answer
|
|
231
|
+
more than one call (`known_values`, `by_period` at several grains, `breakdowns`, a
|
|
232
|
+
`filtered` view each); a single call gains nothing.
|
|
233
|
+
|
|
234
|
+
It is a read-only mapping of the frames it was given. The frames must not change
|
|
235
|
+
while it is in use: what it loaded would no longer be what they hold.
|
|
236
|
+
|
|
237
|
+
Calls from several threads at once are safe. Converting and loading happen under a
|
|
238
|
+
lock, so a column is converted once however many threads ask for it; each call then
|
|
239
|
+
queries through a DuckDB cursor of its own, outside the lock.
|
|
240
|
+
|
|
241
|
+
The database lives until `close()`, the end of a `with` block, or the object being
|
|
242
|
+
garbage collected. Close it while no call is running; a call after that raises
|
|
243
|
+
`ValueError`.
|
|
244
|
+
|
|
245
|
+
>>> with metrics.prepare(frames) as prepared:
|
|
246
|
+
... values = metrics.known_values(semantic, prepared)
|
|
247
|
+
... quarters = metrics.by_period(semantic, prepared, "quarter")
|
|
248
|
+
"""
|
|
249
|
+
|
|
250
|
+
def __init__(self, frames: Mapping[str, pd.DataFrame]) -> None:
|
|
251
|
+
self._frames = dict(frames)
|
|
252
|
+
self._lock = threading.Lock()
|
|
253
|
+
self._connection: Any = None
|
|
254
|
+
self._closed = False
|
|
255
|
+
# Table key -> the database table holding its loaded columns, and those columns.
|
|
256
|
+
self._loaded: dict[str, tuple[str, list[Any]]] = {}
|
|
257
|
+
# Table key -> its tables' row-number column, which no view shows.
|
|
258
|
+
self._rows: dict[str, str] = {}
|
|
259
|
+
self._made = 0
|
|
260
|
+
# Tables a call is reading, and tables replaced by a wider one, dropped when unread.
|
|
261
|
+
self._reading: Counter[str] = Counter()
|
|
262
|
+
self._replaced: set[str] = set()
|
|
263
|
+
|
|
264
|
+
def __getitem__(self, key: str) -> pd.DataFrame:
|
|
265
|
+
return self._frames[key]
|
|
266
|
+
|
|
267
|
+
def __iter__(self) -> Iterator[str]:
|
|
268
|
+
return iter(self._frames)
|
|
269
|
+
|
|
270
|
+
def __len__(self) -> int:
|
|
271
|
+
return len(self._frames)
|
|
272
|
+
|
|
273
|
+
def __repr__(self) -> str:
|
|
274
|
+
state = "closed" if self._closed else f"{len(self._loaded)} tables loaded"
|
|
275
|
+
return f"<PreparedFrames {list(self._frames)!r}, {state}>"
|
|
276
|
+
|
|
277
|
+
def __enter__(self) -> PreparedFrames:
|
|
278
|
+
return self
|
|
279
|
+
|
|
280
|
+
def __exit__(self, *_: object) -> None:
|
|
281
|
+
self.close()
|
|
282
|
+
|
|
283
|
+
@property
|
|
284
|
+
def closed(self) -> bool:
|
|
285
|
+
return self._closed
|
|
286
|
+
|
|
287
|
+
def close(self) -> None:
|
|
288
|
+
"""Close the DuckDB database, and with it everything loaded. Closing twice is fine."""
|
|
289
|
+
with self._lock:
|
|
290
|
+
self._closed = True
|
|
291
|
+
self._loaded.clear()
|
|
292
|
+
if self._connection is not None:
|
|
293
|
+
self._connection.close()
|
|
294
|
+
self._connection = None
|
|
295
|
+
|
|
296
|
+
@contextmanager
|
|
297
|
+
def _relations(self, wanted: Mapping[str, tuple[str, list[Any]]]) -> Iterator[Any]:
|
|
298
|
+
"""A cursor in which each relation name of `wanted` is a view of its table key's
|
|
299
|
+
loaded table, which holds at least the columns asked for."""
|
|
300
|
+
import duckdb
|
|
301
|
+
|
|
302
|
+
with self._lock:
|
|
303
|
+
if self._closed:
|
|
304
|
+
raise ValueError("these prepared frames are closed")
|
|
305
|
+
if self._connection is None:
|
|
306
|
+
self._connection = duckdb.connect(":memory:")
|
|
307
|
+
self._connection.execute("set threads = 1")
|
|
308
|
+
tables = {
|
|
309
|
+
relation: self._table(key, columns) for relation, (key, columns) in wanted.items()
|
|
310
|
+
}
|
|
311
|
+
rows = {relation: self._rows[key] for relation, (key, _) in wanted.items()}
|
|
312
|
+
self._reading.update(tables.values())
|
|
313
|
+
cursor = self._connection.cursor()
|
|
314
|
+
try:
|
|
315
|
+
for relation, table in tables.items():
|
|
316
|
+
# Temporary, so the view is this cursor's alone; it shows the frame's
|
|
317
|
+
# columns only, never the row number.
|
|
318
|
+
cursor.execute(
|
|
319
|
+
f"create temp view {relation} as "
|
|
320
|
+
f"select * exclude ({quote(rows[relation])}) from {table}"
|
|
321
|
+
)
|
|
322
|
+
yield cursor
|
|
323
|
+
finally:
|
|
324
|
+
cursor.close()
|
|
325
|
+
with self._lock:
|
|
326
|
+
self._reading.subtract(tables.values())
|
|
327
|
+
unread = {table for table in self._replaced if self._reading[table] <= 0}
|
|
328
|
+
self._replaced -= unread
|
|
329
|
+
if self._connection is not None:
|
|
330
|
+
for table in sorted(unread):
|
|
331
|
+
self._connection.execute(f"drop table {table}")
|
|
332
|
+
|
|
333
|
+
def _table(self, key: str, columns: list[Any]) -> str:
|
|
334
|
+
"""The table holding `columns` of `key`, converting and loading the ones that are
|
|
335
|
+
not loaded yet. Called under the lock."""
|
|
336
|
+
table, loaded = self._loaded.get(key, ("", []))
|
|
337
|
+
missing = [column for column in columns if column not in loaded]
|
|
338
|
+
if not missing:
|
|
339
|
+
return table
|
|
340
|
+
self._made += 1
|
|
341
|
+
wider = f"m2d_prepared_{self._made}"
|
|
342
|
+
frame = self._frames[key]
|
|
343
|
+
row = self._rows.setdefault(key, _row_column(frame))
|
|
344
|
+
text = _text_columns(frame, missing)
|
|
345
|
+
if not table:
|
|
346
|
+
_load(self._connection, wider, text, row)
|
|
347
|
+
else:
|
|
348
|
+
# Only the new columns are converted; the loaded ones are kept, matched to the
|
|
349
|
+
# new ones by row number and stored in the frame's order, as the first load was.
|
|
350
|
+
_load(self._connection, f"{wider}_new", text, row)
|
|
351
|
+
self._connection.execute(
|
|
352
|
+
f"create table {wider} as select * from {table} "
|
|
353
|
+
f"join {wider}_new using ({quote(row)}) order by {quote(row)}"
|
|
354
|
+
)
|
|
355
|
+
self._connection.execute(f"drop table {wider}_new")
|
|
356
|
+
self._replaced.add(table)
|
|
357
|
+
self._loaded[key] = (wider, [*loaded, *missing])
|
|
358
|
+
return wider
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def prepare(frames: Mapping[str, pd.DataFrame]) -> PreparedFrames:
|
|
362
|
+
"""`frames` (by table key, what the seeds hold) prepared for many metric calls; frames
|
|
363
|
+
already prepared are returned as they are. See `PreparedFrames`."""
|
|
364
|
+
return frames if isinstance(frames, PreparedFrames) else PreparedFrames(frames)
|
|
365
|
+
|
|
366
|
+
|
|
190
367
|
@contextmanager
|
|
191
368
|
def _tables(
|
|
192
369
|
semantic: SemanticModel, frames: Mapping[str, pd.DataFrame], read: Mapping[str, set[str]]
|
|
193
370
|
) -> Iterator[tuple[Any, Callable[[str], str]]]:
|
|
194
371
|
"""A DuckDB connection holding the `read` columns of `frames` as their seeds read,
|
|
195
|
-
and the relation name of each table key.
|
|
196
|
-
import duckdb
|
|
372
|
+
and the relation name of each table key.
|
|
197
373
|
|
|
374
|
+
Plain frames are read for this call alone, into a database of its own. Prepared frames
|
|
375
|
+
give a cursor on their shared database, with the same relation names.
|
|
376
|
+
"""
|
|
198
377
|
keys = [table for table in semantic.entities if table in read]
|
|
199
378
|
for key in keys:
|
|
200
379
|
if key not in frames:
|
|
201
380
|
raise ValueError(f"no generated rows for table {key!r}, which a metric reads")
|
|
202
381
|
names = {key: f"m2d_{index}" for index, key in enumerate(keys)}
|
|
203
382
|
|
|
383
|
+
if isinstance(frames, PreparedFrames):
|
|
384
|
+
wanted = {names[key]: (key, _kept(frames[key], read[key])) for key in keys}
|
|
385
|
+
with frames._relations(wanted) as cursor:
|
|
386
|
+
yield cursor, names.__getitem__
|
|
387
|
+
return
|
|
388
|
+
|
|
389
|
+
import duckdb
|
|
390
|
+
|
|
204
391
|
connection = duckdb.connect(":memory:")
|
|
205
392
|
try:
|
|
206
393
|
connection.execute("set threads = 1")
|
|
207
394
|
for key in keys:
|
|
208
395
|
# One table's text at a time: it is freed once DuckDB holds its own copy.
|
|
209
396
|
text = _text_columns(frames[key], _kept(frames[key], read[key]))
|
|
210
|
-
connection
|
|
211
|
-
columns = ", ".join(
|
|
212
|
-
f"{_NULL_OR_TEXT.format(value=f'cast({quote(c)} as varchar)')} as {quote(c)}"
|
|
213
|
-
for c in text.columns
|
|
214
|
-
)
|
|
215
|
-
connection.execute(
|
|
216
|
-
f"create table {names[key]} as select {columns} from {names[key]}_frame"
|
|
217
|
-
)
|
|
218
|
-
connection.unregister(f"{names[key]}_frame")
|
|
397
|
+
_load(connection, names[key], text)
|
|
219
398
|
del text
|
|
220
399
|
yield connection, names.__getitem__
|
|
221
400
|
finally:
|
|
@@ -175,6 +175,7 @@ tests/test_metrics_dbt.py
|
|
|
175
175
|
tests/test_metrics_expression.py
|
|
176
176
|
tests/test_metrics_lightdash.py
|
|
177
177
|
tests/test_metrics_ossie.py
|
|
178
|
+
tests/test_metrics_prepared.py
|
|
178
179
|
tests/test_metrics_reader.py
|
|
179
180
|
tests/test_metrics_semantic.py
|
|
180
181
|
tests/test_metrics_slices.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "model2data"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.19.0"
|
|
8
8
|
description = "Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values"
|
|
9
9
|
readme = "README_PYPI.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -78,6 +78,15 @@ _case(
|
|
|
78
78
|
"--days",
|
|
79
79
|
"2",
|
|
80
80
|
)
|
|
81
|
+
# The e-commerce example with its own metrics file and the Lightdash meta: recorded on 1.18.0,
|
|
82
|
+
# before metric calls could share prepared frames, which must not change a byte of it.
|
|
83
|
+
_case(
|
|
84
|
+
"metrics-ecommerce",
|
|
85
|
+
_ECOM,
|
|
86
|
+
"--metrics",
|
|
87
|
+
str(EXAMPLES / "ecommerce.metrics.yml"),
|
|
88
|
+
"--lightdash",
|
|
89
|
+
)
|
|
81
90
|
for _locale in ("en_US", "nl_BE", "de_DE", "fr_FR", "ja_JP"):
|
|
82
91
|
_case(f"locale-{_locale}", _ECOM, "--locale", _locale)
|
|
83
92
|
|