dvt-core 1.11.0b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dvt-core might be problematic. Click here for more details.
- dvt/__init__.py +7 -0
- dvt/_pydantic_shim.py +26 -0
- dvt/adapters/__init__.py +16 -0
- dvt/adapters/multi_adapter_manager.py +268 -0
- dvt/artifacts/__init__.py +0 -0
- dvt/artifacts/exceptions/__init__.py +1 -0
- dvt/artifacts/exceptions/schemas.py +31 -0
- dvt/artifacts/resources/__init__.py +116 -0
- dvt/artifacts/resources/base.py +68 -0
- dvt/artifacts/resources/types.py +93 -0
- dvt/artifacts/resources/v1/analysis.py +10 -0
- dvt/artifacts/resources/v1/catalog.py +23 -0
- dvt/artifacts/resources/v1/components.py +275 -0
- dvt/artifacts/resources/v1/config.py +282 -0
- dvt/artifacts/resources/v1/documentation.py +11 -0
- dvt/artifacts/resources/v1/exposure.py +52 -0
- dvt/artifacts/resources/v1/function.py +53 -0
- dvt/artifacts/resources/v1/generic_test.py +32 -0
- dvt/artifacts/resources/v1/group.py +22 -0
- dvt/artifacts/resources/v1/hook.py +11 -0
- dvt/artifacts/resources/v1/macro.py +30 -0
- dvt/artifacts/resources/v1/metric.py +173 -0
- dvt/artifacts/resources/v1/model.py +146 -0
- dvt/artifacts/resources/v1/owner.py +10 -0
- dvt/artifacts/resources/v1/saved_query.py +112 -0
- dvt/artifacts/resources/v1/seed.py +42 -0
- dvt/artifacts/resources/v1/semantic_layer_components.py +72 -0
- dvt/artifacts/resources/v1/semantic_model.py +315 -0
- dvt/artifacts/resources/v1/singular_test.py +14 -0
- dvt/artifacts/resources/v1/snapshot.py +92 -0
- dvt/artifacts/resources/v1/source_definition.py +85 -0
- dvt/artifacts/resources/v1/sql_operation.py +10 -0
- dvt/artifacts/resources/v1/unit_test_definition.py +78 -0
- dvt/artifacts/schemas/__init__.py +0 -0
- dvt/artifacts/schemas/base.py +191 -0
- dvt/artifacts/schemas/batch_results.py +24 -0
- dvt/artifacts/schemas/catalog/__init__.py +12 -0
- dvt/artifacts/schemas/catalog/v1/__init__.py +0 -0
- dvt/artifacts/schemas/catalog/v1/catalog.py +60 -0
- dvt/artifacts/schemas/freshness/__init__.py +1 -0
- dvt/artifacts/schemas/freshness/v3/__init__.py +0 -0
- dvt/artifacts/schemas/freshness/v3/freshness.py +159 -0
- dvt/artifacts/schemas/manifest/__init__.py +2 -0
- dvt/artifacts/schemas/manifest/v12/__init__.py +0 -0
- dvt/artifacts/schemas/manifest/v12/manifest.py +212 -0
- dvt/artifacts/schemas/results.py +148 -0
- dvt/artifacts/schemas/run/__init__.py +2 -0
- dvt/artifacts/schemas/run/v5/__init__.py +0 -0
- dvt/artifacts/schemas/run/v5/run.py +184 -0
- dvt/artifacts/schemas/upgrades/__init__.py +4 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
- dvt/artifacts/utils/validation.py +153 -0
- dvt/cli/__init__.py +1 -0
- dvt/cli/context.py +16 -0
- dvt/cli/exceptions.py +56 -0
- dvt/cli/flags.py +558 -0
- dvt/cli/main.py +971 -0
- dvt/cli/option_types.py +121 -0
- dvt/cli/options.py +79 -0
- dvt/cli/params.py +803 -0
- dvt/cli/requires.py +478 -0
- dvt/cli/resolvers.py +32 -0
- dvt/cli/types.py +40 -0
- dvt/clients/__init__.py +0 -0
- dvt/clients/checked_load.py +82 -0
- dvt/clients/git.py +164 -0
- dvt/clients/jinja.py +206 -0
- dvt/clients/jinja_static.py +245 -0
- dvt/clients/registry.py +192 -0
- dvt/clients/yaml_helper.py +68 -0
- dvt/compilation.py +833 -0
- dvt/compute/__init__.py +26 -0
- dvt/compute/base.py +288 -0
- dvt/compute/engines/__init__.py +13 -0
- dvt/compute/engines/duckdb_engine.py +368 -0
- dvt/compute/engines/spark_engine.py +273 -0
- dvt/compute/query_analyzer.py +212 -0
- dvt/compute/router.py +483 -0
- dvt/config/__init__.py +4 -0
- dvt/config/catalogs.py +95 -0
- dvt/config/compute_config.py +406 -0
- dvt/config/profile.py +411 -0
- dvt/config/profiles_v2.py +464 -0
- dvt/config/project.py +893 -0
- dvt/config/renderer.py +232 -0
- dvt/config/runtime.py +491 -0
- dvt/config/selectors.py +209 -0
- dvt/config/utils.py +78 -0
- dvt/connectors/.gitignore +6 -0
- dvt/connectors/README.md +306 -0
- dvt/connectors/catalog.yml +217 -0
- dvt/connectors/download_connectors.py +300 -0
- dvt/constants.py +29 -0
- dvt/context/__init__.py +0 -0
- dvt/context/base.py +746 -0
- dvt/context/configured.py +136 -0
- dvt/context/context_config.py +350 -0
- dvt/context/docs.py +82 -0
- dvt/context/exceptions_jinja.py +179 -0
- dvt/context/macro_resolver.py +195 -0
- dvt/context/macros.py +171 -0
- dvt/context/manifest.py +73 -0
- dvt/context/providers.py +2198 -0
- dvt/context/query_header.py +14 -0
- dvt/context/secret.py +59 -0
- dvt/context/target.py +74 -0
- dvt/contracts/__init__.py +0 -0
- dvt/contracts/files.py +413 -0
- dvt/contracts/graph/__init__.py +0 -0
- dvt/contracts/graph/manifest.py +1904 -0
- dvt/contracts/graph/metrics.py +98 -0
- dvt/contracts/graph/model_config.py +71 -0
- dvt/contracts/graph/node_args.py +42 -0
- dvt/contracts/graph/nodes.py +1806 -0
- dvt/contracts/graph/semantic_manifest.py +233 -0
- dvt/contracts/graph/unparsed.py +812 -0
- dvt/contracts/project.py +417 -0
- dvt/contracts/results.py +53 -0
- dvt/contracts/selection.py +23 -0
- dvt/contracts/sql.py +86 -0
- dvt/contracts/state.py +69 -0
- dvt/contracts/util.py +46 -0
- dvt/deprecations.py +347 -0
- dvt/deps/__init__.py +0 -0
- dvt/deps/base.py +153 -0
- dvt/deps/git.py +196 -0
- dvt/deps/local.py +80 -0
- dvt/deps/registry.py +131 -0
- dvt/deps/resolver.py +149 -0
- dvt/deps/tarball.py +121 -0
- dvt/docs/source/_ext/dbt_click.py +118 -0
- dvt/docs/source/conf.py +32 -0
- dvt/env_vars.py +64 -0
- dvt/event_time/event_time.py +40 -0
- dvt/event_time/sample_window.py +60 -0
- dvt/events/__init__.py +16 -0
- dvt/events/base_types.py +37 -0
- dvt/events/core_types_pb2.py +2 -0
- dvt/events/logging.py +109 -0
- dvt/events/types.py +2534 -0
- dvt/exceptions.py +1487 -0
- dvt/flags.py +89 -0
- dvt/graph/__init__.py +11 -0
- dvt/graph/cli.py +248 -0
- dvt/graph/graph.py +172 -0
- dvt/graph/queue.py +213 -0
- dvt/graph/selector.py +375 -0
- dvt/graph/selector_methods.py +976 -0
- dvt/graph/selector_spec.py +223 -0
- dvt/graph/thread_pool.py +18 -0
- dvt/hooks.py +21 -0
- dvt/include/README.md +49 -0
- dvt/include/__init__.py +3 -0
- dvt/include/global_project.py +4 -0
- dvt/include/starter_project/.gitignore +4 -0
- dvt/include/starter_project/README.md +15 -0
- dvt/include/starter_project/__init__.py +3 -0
- dvt/include/starter_project/analyses/.gitkeep +0 -0
- dvt/include/starter_project/dvt_project.yml +36 -0
- dvt/include/starter_project/macros/.gitkeep +0 -0
- dvt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
- dvt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
- dvt/include/starter_project/models/example/schema.yml +21 -0
- dvt/include/starter_project/seeds/.gitkeep +0 -0
- dvt/include/starter_project/snapshots/.gitkeep +0 -0
- dvt/include/starter_project/tests/.gitkeep +0 -0
- dvt/internal_deprecations.py +27 -0
- dvt/jsonschemas/__init__.py +3 -0
- dvt/jsonschemas/jsonschemas.py +309 -0
- dvt/jsonschemas/project/0.0.110.json +4717 -0
- dvt/jsonschemas/project/0.0.85.json +2015 -0
- dvt/jsonschemas/resources/0.0.110.json +2636 -0
- dvt/jsonschemas/resources/0.0.85.json +2536 -0
- dvt/jsonschemas/resources/latest.json +6773 -0
- dvt/links.py +4 -0
- dvt/materializations/__init__.py +0 -0
- dvt/materializations/incremental/__init__.py +0 -0
- dvt/materializations/incremental/microbatch.py +235 -0
- dvt/mp_context.py +8 -0
- dvt/node_types.py +37 -0
- dvt/parser/__init__.py +23 -0
- dvt/parser/analysis.py +21 -0
- dvt/parser/base.py +549 -0
- dvt/parser/common.py +267 -0
- dvt/parser/docs.py +52 -0
- dvt/parser/fixtures.py +51 -0
- dvt/parser/functions.py +30 -0
- dvt/parser/generic_test.py +100 -0
- dvt/parser/generic_test_builders.py +334 -0
- dvt/parser/hooks.py +119 -0
- dvt/parser/macros.py +137 -0
- dvt/parser/manifest.py +2204 -0
- dvt/parser/models.py +574 -0
- dvt/parser/partial.py +1179 -0
- dvt/parser/read_files.py +445 -0
- dvt/parser/schema_generic_tests.py +423 -0
- dvt/parser/schema_renderer.py +111 -0
- dvt/parser/schema_yaml_readers.py +936 -0
- dvt/parser/schemas.py +1467 -0
- dvt/parser/search.py +149 -0
- dvt/parser/seeds.py +28 -0
- dvt/parser/singular_test.py +20 -0
- dvt/parser/snapshots.py +44 -0
- dvt/parser/sources.py +557 -0
- dvt/parser/sql.py +63 -0
- dvt/parser/unit_tests.py +622 -0
- dvt/plugins/__init__.py +20 -0
- dvt/plugins/contracts.py +10 -0
- dvt/plugins/exceptions.py +2 -0
- dvt/plugins/manager.py +164 -0
- dvt/plugins/manifest.py +21 -0
- dvt/profiler.py +20 -0
- dvt/py.typed +1 -0
- dvt/runners/__init__.py +2 -0
- dvt/runners/exposure_runner.py +7 -0
- dvt/runners/no_op_runner.py +46 -0
- dvt/runners/saved_query_runner.py +7 -0
- dvt/selected_resources.py +8 -0
- dvt/task/__init__.py +0 -0
- dvt/task/base.py +504 -0
- dvt/task/build.py +197 -0
- dvt/task/clean.py +57 -0
- dvt/task/clone.py +162 -0
- dvt/task/compile.py +151 -0
- dvt/task/compute.py +366 -0
- dvt/task/debug.py +650 -0
- dvt/task/deps.py +280 -0
- dvt/task/docs/__init__.py +3 -0
- dvt/task/docs/generate.py +408 -0
- dvt/task/docs/index.html +250 -0
- dvt/task/docs/serve.py +28 -0
- dvt/task/freshness.py +323 -0
- dvt/task/function.py +122 -0
- dvt/task/group_lookup.py +46 -0
- dvt/task/init.py +374 -0
- dvt/task/list.py +237 -0
- dvt/task/printer.py +176 -0
- dvt/task/profiles.py +256 -0
- dvt/task/retry.py +175 -0
- dvt/task/run.py +1146 -0
- dvt/task/run_operation.py +142 -0
- dvt/task/runnable.py +802 -0
- dvt/task/seed.py +104 -0
- dvt/task/show.py +150 -0
- dvt/task/snapshot.py +57 -0
- dvt/task/sql.py +111 -0
- dvt/task/test.py +464 -0
- dvt/tests/fixtures/__init__.py +1 -0
- dvt/tests/fixtures/project.py +620 -0
- dvt/tests/util.py +651 -0
- dvt/tracking.py +529 -0
- dvt/utils/__init__.py +3 -0
- dvt/utils/artifact_upload.py +151 -0
- dvt/utils/utils.py +408 -0
- dvt/version.py +249 -0
- dvt_core-1.11.0b4.dist-info/METADATA +252 -0
- dvt_core-1.11.0b4.dist-info/RECORD +261 -0
- dvt_core-1.11.0b4.dist-info/WHEEL +5 -0
- dvt_core-1.11.0b4.dist-info/entry_points.txt +2 -0
- dvt_core-1.11.0b4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,936 @@
|
|
|
1
|
+
from collections.abc import Sequence
|
|
2
|
+
from typing import Any, Dict, List, Optional, Union
|
|
3
|
+
|
|
4
|
+
from dvt.artifacts.resources import (
|
|
5
|
+
ConversionTypeParams,
|
|
6
|
+
CumulativeTypeParams,
|
|
7
|
+
Dimension,
|
|
8
|
+
DimensionTypeParams,
|
|
9
|
+
Entity,
|
|
10
|
+
Export,
|
|
11
|
+
ExportConfig,
|
|
12
|
+
ExposureConfig,
|
|
13
|
+
Measure,
|
|
14
|
+
MetricConfig,
|
|
15
|
+
MetricInput,
|
|
16
|
+
MetricInputMeasure,
|
|
17
|
+
MetricTimeWindow,
|
|
18
|
+
MetricTypeParams,
|
|
19
|
+
NonAdditiveDimension,
|
|
20
|
+
QueryParams,
|
|
21
|
+
SavedQueryConfig,
|
|
22
|
+
SemanticLayerElementConfig,
|
|
23
|
+
WhereFilter,
|
|
24
|
+
WhereFilterIntersection,
|
|
25
|
+
)
|
|
26
|
+
from dvt.clients.jinja import get_rendered
|
|
27
|
+
from dvt.context.context_config import (
|
|
28
|
+
BaseContextConfigGenerator,
|
|
29
|
+
ContextConfigGenerator,
|
|
30
|
+
UnrenderedConfigGenerator,
|
|
31
|
+
)
|
|
32
|
+
from dvt.context.providers import (
|
|
33
|
+
generate_parse_exposure,
|
|
34
|
+
generate_parse_semantic_models,
|
|
35
|
+
)
|
|
36
|
+
from dvt.contracts.files import SchemaSourceFile
|
|
37
|
+
from dvt.contracts.graph.nodes import Exposure, Group, Metric, SavedQuery, SemanticModel
|
|
38
|
+
from dvt.contracts.graph.unparsed import (
|
|
39
|
+
UnparsedConversionTypeParams,
|
|
40
|
+
UnparsedCumulativeTypeParams,
|
|
41
|
+
UnparsedDimension,
|
|
42
|
+
UnparsedDimensionTypeParams,
|
|
43
|
+
UnparsedEntity,
|
|
44
|
+
UnparsedExport,
|
|
45
|
+
UnparsedExposure,
|
|
46
|
+
UnparsedGroup,
|
|
47
|
+
UnparsedMeasure,
|
|
48
|
+
UnparsedMetric,
|
|
49
|
+
UnparsedMetricInput,
|
|
50
|
+
UnparsedMetricInputMeasure,
|
|
51
|
+
UnparsedMetricTypeParams,
|
|
52
|
+
UnparsedNonAdditiveDimension,
|
|
53
|
+
UnparsedQueryParams,
|
|
54
|
+
UnparsedSavedQuery,
|
|
55
|
+
UnparsedSemanticModel,
|
|
56
|
+
)
|
|
57
|
+
from dvt.exceptions import JSONValidationError, YamlParseDictError
|
|
58
|
+
from dvt.node_types import NodeType
|
|
59
|
+
from dvt.parser.common import YamlBlock
|
|
60
|
+
from dvt.parser.schemas import ParseResult, SchemaParser, YamlReader
|
|
61
|
+
|
|
62
|
+
from dbt_common.dataclass_schema import ValidationError
|
|
63
|
+
from dbt_common.exceptions import DbtInternalError
|
|
64
|
+
from dbt_semantic_interfaces.type_enums import (
|
|
65
|
+
AggregationType,
|
|
66
|
+
ConversionCalculationType,
|
|
67
|
+
DimensionType,
|
|
68
|
+
EntityType,
|
|
69
|
+
MetricType,
|
|
70
|
+
PeriodAggregation,
|
|
71
|
+
TimeGranularity,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def parse_where_filter(
|
|
76
|
+
where: Optional[Union[List[str], str]]
|
|
77
|
+
) -> Optional[WhereFilterIntersection]:
|
|
78
|
+
if where is None:
|
|
79
|
+
return None
|
|
80
|
+
elif isinstance(where, str):
|
|
81
|
+
return WhereFilterIntersection([WhereFilter(where)])
|
|
82
|
+
else:
|
|
83
|
+
return WhereFilterIntersection([WhereFilter(where_str) for where_str in where])
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class ExposureParser(YamlReader):
|
|
87
|
+
def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None:
|
|
88
|
+
super().__init__(schema_parser, yaml, NodeType.Exposure.pluralize())
|
|
89
|
+
self.schema_parser = schema_parser
|
|
90
|
+
self.yaml = yaml
|
|
91
|
+
|
|
92
|
+
def parse_exposure(self, unparsed: UnparsedExposure) -> None:
|
|
93
|
+
package_name = self.project.project_name
|
|
94
|
+
unique_id = f"{NodeType.Exposure}.{package_name}.{unparsed.name}"
|
|
95
|
+
path = self.yaml.path.relative_path
|
|
96
|
+
|
|
97
|
+
fqn = self.schema_parser.get_fqn_prefix(path)
|
|
98
|
+
fqn.append(unparsed.name)
|
|
99
|
+
|
|
100
|
+
config = self._generate_exposure_config(
|
|
101
|
+
target=unparsed,
|
|
102
|
+
fqn=fqn,
|
|
103
|
+
package_name=package_name,
|
|
104
|
+
rendered=True,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
config = config.finalize_and_validate()
|
|
108
|
+
|
|
109
|
+
unrendered_config = self._generate_exposure_config(
|
|
110
|
+
target=unparsed,
|
|
111
|
+
fqn=fqn,
|
|
112
|
+
package_name=package_name,
|
|
113
|
+
rendered=False,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if not isinstance(config, ExposureConfig):
|
|
117
|
+
raise DbtInternalError(
|
|
118
|
+
f"Calculated a {type(config)} for an exposure, but expected an ExposureConfig"
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
tags = sorted(set(self.project.exposures.get("tags", []) + unparsed.tags + config.tags))
|
|
122
|
+
meta = {**self.project.exposures.get("meta", {}), **unparsed.meta, **config.meta}
|
|
123
|
+
|
|
124
|
+
config.tags = tags
|
|
125
|
+
config.meta = meta
|
|
126
|
+
|
|
127
|
+
parsed = Exposure(
|
|
128
|
+
resource_type=NodeType.Exposure,
|
|
129
|
+
package_name=package_name,
|
|
130
|
+
path=path,
|
|
131
|
+
original_file_path=self.yaml.path.original_file_path,
|
|
132
|
+
unique_id=unique_id,
|
|
133
|
+
fqn=fqn,
|
|
134
|
+
name=unparsed.name,
|
|
135
|
+
type=unparsed.type,
|
|
136
|
+
url=unparsed.url,
|
|
137
|
+
meta=meta,
|
|
138
|
+
tags=tags,
|
|
139
|
+
description=unparsed.description,
|
|
140
|
+
label=unparsed.label,
|
|
141
|
+
owner=unparsed.owner,
|
|
142
|
+
maturity=unparsed.maturity,
|
|
143
|
+
config=config,
|
|
144
|
+
unrendered_config=unrendered_config,
|
|
145
|
+
)
|
|
146
|
+
ctx = generate_parse_exposure(
|
|
147
|
+
parsed,
|
|
148
|
+
self.root_project,
|
|
149
|
+
self.schema_parser.manifest,
|
|
150
|
+
package_name,
|
|
151
|
+
)
|
|
152
|
+
depends_on_jinja = "\n".join("{{ " + line + "}}" for line in unparsed.depends_on)
|
|
153
|
+
get_rendered(depends_on_jinja, ctx, parsed, capture_macros=True)
|
|
154
|
+
# parsed now has a populated refs/sources/metrics
|
|
155
|
+
|
|
156
|
+
assert isinstance(self.yaml.file, SchemaSourceFile)
|
|
157
|
+
if parsed.config.enabled:
|
|
158
|
+
self.manifest.add_exposure(self.yaml.file, parsed)
|
|
159
|
+
else:
|
|
160
|
+
self.manifest.add_disabled(self.yaml.file, parsed)
|
|
161
|
+
|
|
162
|
+
def _generate_exposure_config(
|
|
163
|
+
self, target: UnparsedExposure, fqn: List[str], package_name: str, rendered: bool
|
|
164
|
+
):
|
|
165
|
+
generator: BaseContextConfigGenerator
|
|
166
|
+
if rendered:
|
|
167
|
+
generator = ContextConfigGenerator(self.root_project)
|
|
168
|
+
else:
|
|
169
|
+
generator = UnrenderedConfigGenerator(self.root_project)
|
|
170
|
+
|
|
171
|
+
# configs with precendence set
|
|
172
|
+
precedence_configs = dict()
|
|
173
|
+
# apply exposure configs
|
|
174
|
+
precedence_configs.update(target.config)
|
|
175
|
+
|
|
176
|
+
return generator.calculate_node_config(
|
|
177
|
+
config_call_dict={},
|
|
178
|
+
fqn=fqn,
|
|
179
|
+
resource_type=NodeType.Exposure,
|
|
180
|
+
project_name=package_name,
|
|
181
|
+
base=False,
|
|
182
|
+
patch_config_dict=precedence_configs,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
def parse(self) -> None:
|
|
186
|
+
for data in self.get_key_dicts():
|
|
187
|
+
try:
|
|
188
|
+
UnparsedExposure.validate(data)
|
|
189
|
+
unparsed = UnparsedExposure.from_dict(data)
|
|
190
|
+
except (ValidationError, JSONValidationError) as exc:
|
|
191
|
+
raise YamlParseDictError(self.yaml.path, self.key, data, exc)
|
|
192
|
+
|
|
193
|
+
self.parse_exposure(unparsed)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class MetricParser(YamlReader):
|
|
197
|
+
def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None:
|
|
198
|
+
super().__init__(schema_parser, yaml, NodeType.Metric.pluralize())
|
|
199
|
+
self.schema_parser = schema_parser
|
|
200
|
+
self.yaml = yaml
|
|
201
|
+
|
|
202
|
+
def _get_input_measure(
|
|
203
|
+
self,
|
|
204
|
+
unparsed_input_measure: Union[UnparsedMetricInputMeasure, str],
|
|
205
|
+
) -> MetricInputMeasure:
|
|
206
|
+
if isinstance(unparsed_input_measure, str):
|
|
207
|
+
return MetricInputMeasure(name=unparsed_input_measure)
|
|
208
|
+
else:
|
|
209
|
+
return MetricInputMeasure(
|
|
210
|
+
name=unparsed_input_measure.name,
|
|
211
|
+
filter=parse_where_filter(unparsed_input_measure.filter),
|
|
212
|
+
alias=unparsed_input_measure.alias,
|
|
213
|
+
join_to_timespine=unparsed_input_measure.join_to_timespine,
|
|
214
|
+
fill_nulls_with=unparsed_input_measure.fill_nulls_with,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
def _get_optional_input_measure(
|
|
218
|
+
self,
|
|
219
|
+
unparsed_input_measure: Optional[Union[UnparsedMetricInputMeasure, str]],
|
|
220
|
+
) -> Optional[MetricInputMeasure]:
|
|
221
|
+
if unparsed_input_measure is not None:
|
|
222
|
+
return self._get_input_measure(unparsed_input_measure)
|
|
223
|
+
else:
|
|
224
|
+
return None
|
|
225
|
+
|
|
226
|
+
def _get_input_measures(
|
|
227
|
+
self,
|
|
228
|
+
unparsed_input_measures: Optional[List[Union[UnparsedMetricInputMeasure, str]]],
|
|
229
|
+
) -> List[MetricInputMeasure]:
|
|
230
|
+
input_measures: List[MetricInputMeasure] = []
|
|
231
|
+
if unparsed_input_measures is not None:
|
|
232
|
+
for unparsed_input_measure in unparsed_input_measures:
|
|
233
|
+
input_measures.append(self._get_input_measure(unparsed_input_measure))
|
|
234
|
+
|
|
235
|
+
return input_measures
|
|
236
|
+
|
|
237
|
+
def _get_period_agg(self, unparsed_period_agg: str) -> PeriodAggregation:
|
|
238
|
+
return PeriodAggregation(unparsed_period_agg)
|
|
239
|
+
|
|
240
|
+
def _get_optional_time_window(
|
|
241
|
+
self, unparsed_window: Optional[str]
|
|
242
|
+
) -> Optional[MetricTimeWindow]:
|
|
243
|
+
if unparsed_window is not None:
|
|
244
|
+
parts = unparsed_window.lower().split(" ")
|
|
245
|
+
if len(parts) != 2:
|
|
246
|
+
raise YamlParseDictError(
|
|
247
|
+
self.yaml.path,
|
|
248
|
+
"window",
|
|
249
|
+
{"window": unparsed_window},
|
|
250
|
+
f"Invalid window ({unparsed_window}) in cumulative/conversion metric. Should be of the form `<count> <granularity>`, "
|
|
251
|
+
"e.g., `28 days`",
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
granularity = parts[1]
|
|
255
|
+
# once we drop python 3.8 this could just be `granularity = parts[0].removesuffix('s')
|
|
256
|
+
if granularity.endswith("s") and granularity[:-1] in [
|
|
257
|
+
item.value for item in TimeGranularity
|
|
258
|
+
]:
|
|
259
|
+
# Can only remove the `s` if it's a standard grain, months -> month
|
|
260
|
+
granularity = granularity[:-1]
|
|
261
|
+
|
|
262
|
+
count = parts[0]
|
|
263
|
+
if not count.isdigit():
|
|
264
|
+
raise YamlParseDictError(
|
|
265
|
+
self.yaml.path,
|
|
266
|
+
"window",
|
|
267
|
+
{"window": unparsed_window},
|
|
268
|
+
f"Invalid count ({count}) in cumulative/conversion metric window string: ({unparsed_window})",
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
return MetricTimeWindow(
|
|
272
|
+
count=int(count),
|
|
273
|
+
granularity=granularity,
|
|
274
|
+
)
|
|
275
|
+
else:
|
|
276
|
+
return None
|
|
277
|
+
|
|
278
|
+
def _get_metric_input(self, unparsed: Union[UnparsedMetricInput, str]) -> MetricInput:
|
|
279
|
+
if isinstance(unparsed, str):
|
|
280
|
+
return MetricInput(name=unparsed)
|
|
281
|
+
else:
|
|
282
|
+
return MetricInput(
|
|
283
|
+
name=unparsed.name,
|
|
284
|
+
filter=parse_where_filter(unparsed.filter),
|
|
285
|
+
alias=unparsed.alias,
|
|
286
|
+
offset_window=self._get_optional_time_window(unparsed.offset_window),
|
|
287
|
+
offset_to_grain=unparsed.offset_to_grain,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
def _get_optional_metric_input(
|
|
291
|
+
self,
|
|
292
|
+
unparsed: Optional[Union[UnparsedMetricInput, str]],
|
|
293
|
+
) -> Optional[MetricInput]:
|
|
294
|
+
if unparsed is not None:
|
|
295
|
+
return self._get_metric_input(unparsed)
|
|
296
|
+
else:
|
|
297
|
+
return None
|
|
298
|
+
|
|
299
|
+
def _get_metric_inputs(
|
|
300
|
+
self,
|
|
301
|
+
unparsed_metric_inputs: Optional[List[Union[UnparsedMetricInput, str]]],
|
|
302
|
+
) -> List[MetricInput]:
|
|
303
|
+
metric_inputs: List[MetricInput] = []
|
|
304
|
+
if unparsed_metric_inputs is not None:
|
|
305
|
+
for unparsed_metric_input in unparsed_metric_inputs:
|
|
306
|
+
metric_inputs.append(self._get_metric_input(unparsed=unparsed_metric_input))
|
|
307
|
+
|
|
308
|
+
return metric_inputs
|
|
309
|
+
|
|
310
|
+
def _get_optional_conversion_type_params(
|
|
311
|
+
self, unparsed: Optional[UnparsedConversionTypeParams]
|
|
312
|
+
) -> Optional[ConversionTypeParams]:
|
|
313
|
+
if unparsed is None:
|
|
314
|
+
return None
|
|
315
|
+
return ConversionTypeParams(
|
|
316
|
+
base_measure=self._get_input_measure(unparsed.base_measure),
|
|
317
|
+
conversion_measure=self._get_input_measure(unparsed.conversion_measure),
|
|
318
|
+
entity=unparsed.entity,
|
|
319
|
+
calculation=ConversionCalculationType(unparsed.calculation),
|
|
320
|
+
window=self._get_optional_time_window(unparsed.window),
|
|
321
|
+
constant_properties=unparsed.constant_properties,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
def _get_optional_cumulative_type_params(
|
|
325
|
+
self, unparsed_metric: UnparsedMetric
|
|
326
|
+
) -> Optional[CumulativeTypeParams]:
|
|
327
|
+
unparsed_type_params = unparsed_metric.type_params
|
|
328
|
+
if unparsed_metric.type.lower() == MetricType.CUMULATIVE.value:
|
|
329
|
+
if not unparsed_type_params.cumulative_type_params:
|
|
330
|
+
unparsed_type_params.cumulative_type_params = UnparsedCumulativeTypeParams()
|
|
331
|
+
|
|
332
|
+
if (
|
|
333
|
+
unparsed_type_params.window
|
|
334
|
+
and not unparsed_type_params.cumulative_type_params.window
|
|
335
|
+
):
|
|
336
|
+
unparsed_type_params.cumulative_type_params.window = unparsed_type_params.window
|
|
337
|
+
if (
|
|
338
|
+
unparsed_type_params.grain_to_date
|
|
339
|
+
and not unparsed_type_params.cumulative_type_params.grain_to_date
|
|
340
|
+
):
|
|
341
|
+
unparsed_type_params.cumulative_type_params.grain_to_date = (
|
|
342
|
+
unparsed_type_params.grain_to_date
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
return CumulativeTypeParams(
|
|
346
|
+
window=self._get_optional_time_window(
|
|
347
|
+
unparsed_type_params.cumulative_type_params.window
|
|
348
|
+
),
|
|
349
|
+
grain_to_date=unparsed_type_params.cumulative_type_params.grain_to_date,
|
|
350
|
+
period_agg=self._get_period_agg(
|
|
351
|
+
unparsed_type_params.cumulative_type_params.period_agg
|
|
352
|
+
),
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
return None
|
|
356
|
+
|
|
357
|
+
def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricTypeParams:
|
|
358
|
+
type_params = unparsed_metric.type_params
|
|
359
|
+
|
|
360
|
+
grain_to_date: Optional[TimeGranularity] = None
|
|
361
|
+
if type_params.grain_to_date is not None:
|
|
362
|
+
# This should've been changed to a string (to support custom grain), but since this
|
|
363
|
+
# is a legacy field waiting to be deprecated, we will not support custom grain here
|
|
364
|
+
# in order to force customers off of using this field. The field to use should be
|
|
365
|
+
# `cumulative_type_params.grain_to_date`
|
|
366
|
+
grain_to_date = TimeGranularity(type_params.grain_to_date)
|
|
367
|
+
|
|
368
|
+
return MetricTypeParams(
|
|
369
|
+
measure=self._get_optional_input_measure(type_params.measure),
|
|
370
|
+
numerator=self._get_optional_metric_input(type_params.numerator),
|
|
371
|
+
denominator=self._get_optional_metric_input(type_params.denominator),
|
|
372
|
+
expr=str(type_params.expr) if type_params.expr is not None else None,
|
|
373
|
+
window=self._get_optional_time_window(type_params.window),
|
|
374
|
+
grain_to_date=grain_to_date,
|
|
375
|
+
metrics=self._get_metric_inputs(type_params.metrics),
|
|
376
|
+
conversion_type_params=self._get_optional_conversion_type_params(
|
|
377
|
+
type_params.conversion_type_params
|
|
378
|
+
),
|
|
379
|
+
cumulative_type_params=self._get_optional_cumulative_type_params(
|
|
380
|
+
unparsed_metric=unparsed_metric,
|
|
381
|
+
),
|
|
382
|
+
# input measures are calculated via metric processing post parsing
|
|
383
|
+
# input_measures=?,
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
def parse_metric(self, unparsed: UnparsedMetric, generated_from: Optional[str] = None) -> None:
|
|
387
|
+
package_name = self.project.project_name
|
|
388
|
+
unique_id = f"{NodeType.Metric}.{package_name}.{unparsed.name}"
|
|
389
|
+
path = self.yaml.path.relative_path
|
|
390
|
+
|
|
391
|
+
fqn = self.schema_parser.get_fqn_prefix(path)
|
|
392
|
+
fqn.append(unparsed.name)
|
|
393
|
+
|
|
394
|
+
config = self._generate_metric_config(
|
|
395
|
+
target=unparsed,
|
|
396
|
+
fqn=fqn,
|
|
397
|
+
package_name=package_name,
|
|
398
|
+
rendered=True,
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
config = config.finalize_and_validate()
|
|
402
|
+
|
|
403
|
+
unrendered_config = self._generate_metric_config(
|
|
404
|
+
target=unparsed,
|
|
405
|
+
fqn=fqn,
|
|
406
|
+
package_name=package_name,
|
|
407
|
+
rendered=False,
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
if not isinstance(config, MetricConfig):
|
|
411
|
+
raise DbtInternalError(
|
|
412
|
+
f"Calculated a {type(config)} for a metric, but expected a MetricConfig"
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
# If we have meta in the config, copy to node level, for backwards
|
|
416
|
+
# compatibility with earlier node-only config.
|
|
417
|
+
if "meta" in config and config["meta"]:
|
|
418
|
+
unparsed.meta = config["meta"]
|
|
419
|
+
|
|
420
|
+
parsed = Metric(
|
|
421
|
+
resource_type=NodeType.Metric,
|
|
422
|
+
package_name=package_name,
|
|
423
|
+
path=path,
|
|
424
|
+
original_file_path=self.yaml.path.original_file_path,
|
|
425
|
+
unique_id=unique_id,
|
|
426
|
+
fqn=fqn,
|
|
427
|
+
name=unparsed.name,
|
|
428
|
+
description=unparsed.description,
|
|
429
|
+
label=unparsed.label,
|
|
430
|
+
type=MetricType(unparsed.type),
|
|
431
|
+
type_params=self._get_metric_type_params(unparsed),
|
|
432
|
+
time_granularity=unparsed.time_granularity,
|
|
433
|
+
filter=parse_where_filter(unparsed.filter),
|
|
434
|
+
meta=unparsed.meta,
|
|
435
|
+
tags=unparsed.tags,
|
|
436
|
+
config=config,
|
|
437
|
+
unrendered_config=unrendered_config,
|
|
438
|
+
group=config.group,
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
# if the metric is disabled we do not want it included in the manifest, only in the disabled dict
|
|
442
|
+
assert isinstance(self.yaml.file, SchemaSourceFile)
|
|
443
|
+
if parsed.config.enabled:
|
|
444
|
+
self.manifest.add_metric(self.yaml.file, parsed, generated_from)
|
|
445
|
+
else:
|
|
446
|
+
self.manifest.add_disabled(self.yaml.file, parsed)
|
|
447
|
+
|
|
448
|
+
def _generate_metric_config(
|
|
449
|
+
self, target: UnparsedMetric, fqn: List[str], package_name: str, rendered: bool
|
|
450
|
+
):
|
|
451
|
+
generator: BaseContextConfigGenerator
|
|
452
|
+
if rendered:
|
|
453
|
+
generator = ContextConfigGenerator(self.root_project)
|
|
454
|
+
else:
|
|
455
|
+
generator = UnrenderedConfigGenerator(self.root_project)
|
|
456
|
+
|
|
457
|
+
# configs with precendence set
|
|
458
|
+
precedence_configs = dict()
|
|
459
|
+
# first apply metric configs
|
|
460
|
+
precedence_configs.update(target.config)
|
|
461
|
+
|
|
462
|
+
config = generator.calculate_node_config(
|
|
463
|
+
config_call_dict={},
|
|
464
|
+
fqn=fqn,
|
|
465
|
+
resource_type=NodeType.Metric,
|
|
466
|
+
project_name=package_name,
|
|
467
|
+
base=False,
|
|
468
|
+
patch_config_dict=precedence_configs,
|
|
469
|
+
)
|
|
470
|
+
return config
|
|
471
|
+
|
|
472
|
+
def parse(self) -> None:
|
|
473
|
+
for data in self.get_key_dicts():
|
|
474
|
+
try:
|
|
475
|
+
UnparsedMetric.validate(data)
|
|
476
|
+
unparsed = UnparsedMetric.from_dict(data)
|
|
477
|
+
|
|
478
|
+
except (ValidationError, JSONValidationError) as exc:
|
|
479
|
+
raise YamlParseDictError(self.yaml.path, self.key, data, exc)
|
|
480
|
+
self.parse_metric(unparsed)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
class GroupParser(YamlReader):
|
|
484
|
+
def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None:
|
|
485
|
+
super().__init__(schema_parser, yaml, NodeType.Group.pluralize())
|
|
486
|
+
self.schema_parser = schema_parser
|
|
487
|
+
self.yaml = yaml
|
|
488
|
+
|
|
489
|
+
def parse_group(self, unparsed: UnparsedGroup) -> None:
|
|
490
|
+
package_name = self.project.project_name
|
|
491
|
+
unique_id = f"{NodeType.Group}.{package_name}.{unparsed.name}"
|
|
492
|
+
path = self.yaml.path.relative_path
|
|
493
|
+
|
|
494
|
+
fqn = self.schema_parser.get_fqn_prefix(path)
|
|
495
|
+
fqn.append(unparsed.name)
|
|
496
|
+
config = self._generate_group_config(unparsed, fqn, package_name, True)
|
|
497
|
+
|
|
498
|
+
parsed = Group(
|
|
499
|
+
resource_type=NodeType.Group,
|
|
500
|
+
package_name=package_name,
|
|
501
|
+
path=path,
|
|
502
|
+
original_file_path=self.yaml.path.original_file_path,
|
|
503
|
+
unique_id=unique_id,
|
|
504
|
+
name=unparsed.name,
|
|
505
|
+
owner=unparsed.owner,
|
|
506
|
+
description=unparsed.description,
|
|
507
|
+
config=config,
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
assert isinstance(self.yaml.file, SchemaSourceFile)
|
|
511
|
+
self.manifest.add_group(self.yaml.file, parsed)
|
|
512
|
+
|
|
513
|
+
def parse(self):
|
|
514
|
+
for data in self.get_key_dicts():
|
|
515
|
+
try:
|
|
516
|
+
UnparsedGroup.validate(data)
|
|
517
|
+
unparsed = UnparsedGroup.from_dict(data)
|
|
518
|
+
except (ValidationError, JSONValidationError) as exc:
|
|
519
|
+
raise YamlParseDictError(self.yaml.path, self.key, data, exc)
|
|
520
|
+
|
|
521
|
+
self.parse_group(unparsed)
|
|
522
|
+
|
|
523
|
+
def _generate_group_config(
|
|
524
|
+
self, target: UnparsedGroup, fqn: List[str], package_name: str, rendered: bool
|
|
525
|
+
):
|
|
526
|
+
generator: BaseContextConfigGenerator
|
|
527
|
+
if rendered:
|
|
528
|
+
generator = ContextConfigGenerator(self.root_project)
|
|
529
|
+
else:
|
|
530
|
+
generator = UnrenderedConfigGenerator(self.root_project)
|
|
531
|
+
|
|
532
|
+
# configs with precendence set
|
|
533
|
+
precedence_configs = dict()
|
|
534
|
+
# first apply metric configs
|
|
535
|
+
precedence_configs.update(target.config)
|
|
536
|
+
|
|
537
|
+
config = generator.calculate_node_config(
|
|
538
|
+
config_call_dict={},
|
|
539
|
+
fqn=fqn,
|
|
540
|
+
resource_type=NodeType.Group,
|
|
541
|
+
project_name=package_name,
|
|
542
|
+
base=False,
|
|
543
|
+
patch_config_dict=precedence_configs,
|
|
544
|
+
)
|
|
545
|
+
return config
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
class SemanticModelParser(YamlReader):
|
|
549
|
+
def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None:
|
|
550
|
+
super().__init__(schema_parser, yaml, "semantic_models")
|
|
551
|
+
self.schema_parser = schema_parser
|
|
552
|
+
self.yaml = yaml
|
|
553
|
+
|
|
554
|
+
def _get_dimension_type_params(
|
|
555
|
+
self, unparsed: Optional[UnparsedDimensionTypeParams]
|
|
556
|
+
) -> Optional[DimensionTypeParams]:
|
|
557
|
+
if unparsed is not None:
|
|
558
|
+
return DimensionTypeParams(
|
|
559
|
+
time_granularity=TimeGranularity(unparsed.time_granularity),
|
|
560
|
+
validity_params=unparsed.validity_params,
|
|
561
|
+
)
|
|
562
|
+
else:
|
|
563
|
+
return None
|
|
564
|
+
|
|
565
|
+
def _get_dimensions(self, unparsed_dimensions: List[UnparsedDimension]) -> List[Dimension]:
|
|
566
|
+
dimensions: List[Dimension] = []
|
|
567
|
+
for unparsed in unparsed_dimensions:
|
|
568
|
+
dimensions.append(
|
|
569
|
+
Dimension(
|
|
570
|
+
name=unparsed.name,
|
|
571
|
+
type=DimensionType(unparsed.type),
|
|
572
|
+
description=unparsed.description,
|
|
573
|
+
label=unparsed.label,
|
|
574
|
+
is_partition=unparsed.is_partition,
|
|
575
|
+
type_params=self._get_dimension_type_params(unparsed=unparsed.type_params),
|
|
576
|
+
expr=unparsed.expr,
|
|
577
|
+
metadata=None, # TODO: requires a fair bit of parsing context
|
|
578
|
+
config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})),
|
|
579
|
+
)
|
|
580
|
+
)
|
|
581
|
+
return dimensions
|
|
582
|
+
|
|
583
|
+
def _get_entities(self, unparsed_entities: List[UnparsedEntity]) -> List[Entity]:
|
|
584
|
+
entities: List[Entity] = []
|
|
585
|
+
for unparsed in unparsed_entities:
|
|
586
|
+
entities.append(
|
|
587
|
+
Entity(
|
|
588
|
+
name=unparsed.name,
|
|
589
|
+
type=EntityType(unparsed.type),
|
|
590
|
+
description=unparsed.description,
|
|
591
|
+
label=unparsed.label,
|
|
592
|
+
role=unparsed.role,
|
|
593
|
+
expr=unparsed.expr,
|
|
594
|
+
config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})),
|
|
595
|
+
)
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
return entities
|
|
599
|
+
|
|
600
|
+
def _get_non_additive_dimension(
|
|
601
|
+
self, unparsed: Optional[UnparsedNonAdditiveDimension]
|
|
602
|
+
) -> Optional[NonAdditiveDimension]:
|
|
603
|
+
if unparsed is not None:
|
|
604
|
+
return NonAdditiveDimension(
|
|
605
|
+
name=unparsed.name,
|
|
606
|
+
window_choice=AggregationType(unparsed.window_choice),
|
|
607
|
+
window_groupings=unparsed.window_groupings,
|
|
608
|
+
)
|
|
609
|
+
else:
|
|
610
|
+
return None
|
|
611
|
+
|
|
612
|
+
def _get_measures(self, unparsed_measures: List[UnparsedMeasure]) -> List[Measure]:
|
|
613
|
+
measures: List[Measure] = []
|
|
614
|
+
for unparsed in unparsed_measures:
|
|
615
|
+
measures.append(
|
|
616
|
+
Measure(
|
|
617
|
+
name=unparsed.name,
|
|
618
|
+
agg=AggregationType(unparsed.agg),
|
|
619
|
+
description=unparsed.description,
|
|
620
|
+
label=unparsed.label,
|
|
621
|
+
expr=str(unparsed.expr) if unparsed.expr is not None else None,
|
|
622
|
+
agg_params=unparsed.agg_params,
|
|
623
|
+
non_additive_dimension=self._get_non_additive_dimension(
|
|
624
|
+
unparsed.non_additive_dimension
|
|
625
|
+
),
|
|
626
|
+
agg_time_dimension=unparsed.agg_time_dimension,
|
|
627
|
+
config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})),
|
|
628
|
+
)
|
|
629
|
+
)
|
|
630
|
+
return measures
|
|
631
|
+
|
|
632
|
+
def _create_metric(
|
|
633
|
+
self,
|
|
634
|
+
measure: UnparsedMeasure,
|
|
635
|
+
enabled: bool,
|
|
636
|
+
semantic_model_name: str,
|
|
637
|
+
meta: Optional[Dict[str, Any]] = None,
|
|
638
|
+
) -> None:
|
|
639
|
+
config: Dict[str, Any] = {"enabled": enabled}
|
|
640
|
+
if meta is not None:
|
|
641
|
+
# Need to propagate meta to metric from measure during create_metric: True
|
|
642
|
+
config["meta"] = meta
|
|
643
|
+
unparsed_metric = UnparsedMetric(
|
|
644
|
+
name=measure.name,
|
|
645
|
+
label=measure.label or measure.name,
|
|
646
|
+
type="simple",
|
|
647
|
+
type_params=UnparsedMetricTypeParams(
|
|
648
|
+
measure=measure.name, expr=measure.expr or measure.name # type: ignore
|
|
649
|
+
),
|
|
650
|
+
description=measure.description or f"Metric created from measure {measure.name}",
|
|
651
|
+
config=config,
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
parser = MetricParser(self.schema_parser, yaml=self.yaml)
|
|
655
|
+
parser.parse_metric(unparsed=unparsed_metric, generated_from=semantic_model_name)
|
|
656
|
+
|
|
657
|
+
def _generate_semantic_model_config(
|
|
658
|
+
self, target: UnparsedSemanticModel, fqn: List[str], package_name: str, rendered: bool
|
|
659
|
+
):
|
|
660
|
+
generator: BaseContextConfigGenerator
|
|
661
|
+
if rendered:
|
|
662
|
+
generator = ContextConfigGenerator(self.root_project)
|
|
663
|
+
else:
|
|
664
|
+
generator = UnrenderedConfigGenerator(self.root_project)
|
|
665
|
+
|
|
666
|
+
# configs with precendence set
|
|
667
|
+
precedence_configs = dict()
|
|
668
|
+
# first apply semantic model configs
|
|
669
|
+
precedence_configs.update(target.config)
|
|
670
|
+
|
|
671
|
+
config = generator.calculate_node_config(
|
|
672
|
+
config_call_dict={},
|
|
673
|
+
fqn=fqn,
|
|
674
|
+
resource_type=NodeType.SemanticModel,
|
|
675
|
+
project_name=package_name,
|
|
676
|
+
base=False,
|
|
677
|
+
patch_config_dict=precedence_configs,
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
return config
|
|
681
|
+
|
|
682
|
+
def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None:
|
|
683
|
+
package_name = self.project.project_name
|
|
684
|
+
unique_id = f"{NodeType.SemanticModel}.{package_name}.{unparsed.name}"
|
|
685
|
+
path = self.yaml.path.relative_path
|
|
686
|
+
|
|
687
|
+
fqn = self.schema_parser.get_fqn_prefix(path)
|
|
688
|
+
fqn.append(unparsed.name)
|
|
689
|
+
|
|
690
|
+
entities = self._get_entities(unparsed.entities)
|
|
691
|
+
measures = self._get_measures(unparsed.measures)
|
|
692
|
+
dimensions = self._get_dimensions(unparsed.dimensions)
|
|
693
|
+
|
|
694
|
+
config = self._generate_semantic_model_config(
|
|
695
|
+
target=unparsed,
|
|
696
|
+
fqn=fqn,
|
|
697
|
+
package_name=package_name,
|
|
698
|
+
rendered=True,
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
# Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs
|
|
702
|
+
elements: Sequence[Union[Dimension, Entity, Measure]] = [
|
|
703
|
+
*dimensions,
|
|
704
|
+
*entities,
|
|
705
|
+
*measures,
|
|
706
|
+
]
|
|
707
|
+
for element in elements:
|
|
708
|
+
if config is not None:
|
|
709
|
+
if element.config is None:
|
|
710
|
+
element.config = SemanticLayerElementConfig(meta=config.meta)
|
|
711
|
+
else:
|
|
712
|
+
element.config.meta = {**config.get("meta", {}), **element.config.meta}
|
|
713
|
+
|
|
714
|
+
config = config.finalize_and_validate()
|
|
715
|
+
|
|
716
|
+
unrendered_config = self._generate_semantic_model_config(
|
|
717
|
+
target=unparsed,
|
|
718
|
+
fqn=fqn,
|
|
719
|
+
package_name=package_name,
|
|
720
|
+
rendered=False,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
parsed = SemanticModel(
|
|
724
|
+
description=unparsed.description,
|
|
725
|
+
label=unparsed.label,
|
|
726
|
+
fqn=fqn,
|
|
727
|
+
model=unparsed.model,
|
|
728
|
+
name=unparsed.name,
|
|
729
|
+
node_relation=None, # Resolved from the value of "model" after parsing
|
|
730
|
+
original_file_path=self.yaml.path.original_file_path,
|
|
731
|
+
package_name=package_name,
|
|
732
|
+
path=path,
|
|
733
|
+
resource_type=NodeType.SemanticModel,
|
|
734
|
+
unique_id=unique_id,
|
|
735
|
+
entities=entities,
|
|
736
|
+
measures=measures,
|
|
737
|
+
dimensions=dimensions,
|
|
738
|
+
defaults=unparsed.defaults,
|
|
739
|
+
primary_entity=unparsed.primary_entity,
|
|
740
|
+
config=config,
|
|
741
|
+
unrendered_config=unrendered_config,
|
|
742
|
+
group=config.group,
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
ctx = generate_parse_semantic_models(
|
|
746
|
+
parsed,
|
|
747
|
+
self.root_project,
|
|
748
|
+
self.schema_parser.manifest,
|
|
749
|
+
package_name,
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
if parsed.model is not None:
|
|
753
|
+
model_ref = "{{ " + parsed.model + " }}"
|
|
754
|
+
# This sets the "refs" in the SemanticModel from the SemanticModelRefResolver in context/providers.py
|
|
755
|
+
get_rendered(model_ref, ctx, parsed)
|
|
756
|
+
|
|
757
|
+
# if the semantic model is disabled we do not want it included in the manifest,
|
|
758
|
+
# only in the disabled dict
|
|
759
|
+
assert isinstance(self.yaml.file, SchemaSourceFile)
|
|
760
|
+
if parsed.config.enabled:
|
|
761
|
+
self.manifest.add_semantic_model(self.yaml.file, parsed)
|
|
762
|
+
else:
|
|
763
|
+
self.manifest.add_disabled(self.yaml.file, parsed)
|
|
764
|
+
|
|
765
|
+
# Create a metric for each measure with `create_metric = True`
|
|
766
|
+
for measure in unparsed.measures:
|
|
767
|
+
if measure.create_metric is True:
|
|
768
|
+
self._create_metric(
|
|
769
|
+
measure=measure,
|
|
770
|
+
enabled=parsed.config.enabled,
|
|
771
|
+
semantic_model_name=parsed.name,
|
|
772
|
+
meta=config.meta if config is not None else None,
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
def parse(self) -> None:
|
|
776
|
+
for data in self.get_key_dicts():
|
|
777
|
+
try:
|
|
778
|
+
UnparsedSemanticModel.validate(data)
|
|
779
|
+
unparsed = UnparsedSemanticModel.from_dict(data)
|
|
780
|
+
except (ValidationError, JSONValidationError) as exc:
|
|
781
|
+
raise YamlParseDictError(self.yaml.path, self.key, data, exc)
|
|
782
|
+
|
|
783
|
+
self.parse_semantic_model(unparsed)
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
class SavedQueryParser(YamlReader):
|
|
787
|
+
def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None:
|
|
788
|
+
super().__init__(schema_parser, yaml, "saved_queries")
|
|
789
|
+
self.schema_parser = schema_parser
|
|
790
|
+
self.yaml = yaml
|
|
791
|
+
|
|
792
|
+
def _generate_saved_query_config(
|
|
793
|
+
self, target: UnparsedSavedQuery, fqn: List[str], package_name: str, rendered: bool
|
|
794
|
+
):
|
|
795
|
+
generator: BaseContextConfigGenerator
|
|
796
|
+
if rendered:
|
|
797
|
+
generator = ContextConfigGenerator(self.root_project)
|
|
798
|
+
else:
|
|
799
|
+
generator = UnrenderedConfigGenerator(self.root_project)
|
|
800
|
+
|
|
801
|
+
# configs with precendence set
|
|
802
|
+
precedence_configs = dict()
|
|
803
|
+
# first apply semantic model configs
|
|
804
|
+
precedence_configs.update(target.config)
|
|
805
|
+
|
|
806
|
+
config = generator.calculate_node_config(
|
|
807
|
+
config_call_dict={},
|
|
808
|
+
fqn=fqn,
|
|
809
|
+
resource_type=NodeType.SavedQuery,
|
|
810
|
+
project_name=package_name,
|
|
811
|
+
base=False,
|
|
812
|
+
patch_config_dict=precedence_configs,
|
|
813
|
+
)
|
|
814
|
+
|
|
815
|
+
return config
|
|
816
|
+
|
|
817
|
+
def _get_export_config(
|
|
818
|
+
self, unparsed_export_config: Dict[str, Any], saved_query_config: SavedQueryConfig
|
|
819
|
+
) -> ExportConfig:
|
|
820
|
+
# Combine the two dictionaries using dictionary unpacking
|
|
821
|
+
# the second dictionary is the one whose keys take priority
|
|
822
|
+
combined = {**saved_query_config.__dict__, **unparsed_export_config}
|
|
823
|
+
# `schema` is the user facing attribute, but for DSI protocol purposes we track it as `schema_name`
|
|
824
|
+
if combined.get("schema") is not None and combined.get("schema_name") is None:
|
|
825
|
+
combined["schema_name"] = combined["schema"]
|
|
826
|
+
|
|
827
|
+
return ExportConfig.from_dict(combined)
|
|
828
|
+
|
|
829
|
+
def _get_export(
|
|
830
|
+
self, unparsed: UnparsedExport, saved_query_config: SavedQueryConfig
|
|
831
|
+
) -> Export:
|
|
832
|
+
return Export(
|
|
833
|
+
name=unparsed.name,
|
|
834
|
+
config=self._get_export_config(unparsed.config, saved_query_config),
|
|
835
|
+
unrendered_config=unparsed.config,
|
|
836
|
+
)
|
|
837
|
+
|
|
838
|
+
def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams:
|
|
839
|
+
return QueryParams(
|
|
840
|
+
group_by=unparsed.group_by,
|
|
841
|
+
metrics=unparsed.metrics,
|
|
842
|
+
where=parse_where_filter(unparsed.where),
|
|
843
|
+
order_by=unparsed.order_by,
|
|
844
|
+
limit=unparsed.limit,
|
|
845
|
+
)
|
|
846
|
+
|
|
847
|
+
def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None:
|
|
848
|
+
package_name = self.project.project_name
|
|
849
|
+
unique_id = f"{NodeType.SavedQuery}.{package_name}.{unparsed.name}"
|
|
850
|
+
path = self.yaml.path.relative_path
|
|
851
|
+
|
|
852
|
+
fqn = self.schema_parser.get_fqn_prefix(path)
|
|
853
|
+
fqn.append(unparsed.name)
|
|
854
|
+
|
|
855
|
+
config = self._generate_saved_query_config(
|
|
856
|
+
target=unparsed,
|
|
857
|
+
fqn=fqn,
|
|
858
|
+
package_name=package_name,
|
|
859
|
+
rendered=True,
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
config = config.finalize_and_validate()
|
|
863
|
+
|
|
864
|
+
unrendered_config = self._generate_saved_query_config(
|
|
865
|
+
target=unparsed,
|
|
866
|
+
fqn=fqn,
|
|
867
|
+
package_name=package_name,
|
|
868
|
+
rendered=False,
|
|
869
|
+
)
|
|
870
|
+
|
|
871
|
+
# The parser handles plain strings just fine, but we need to be able
|
|
872
|
+
# to join two lists, remove duplicates, and sort, so we have to wrap things here.
|
|
873
|
+
def wrap_tags(s: Union[List[str], str]) -> List[str]:
|
|
874
|
+
if s is None:
|
|
875
|
+
return []
|
|
876
|
+
return [s] if isinstance(s, str) else s
|
|
877
|
+
|
|
878
|
+
config_tags = wrap_tags(config.get("tags"))
|
|
879
|
+
unparsed_tags = wrap_tags(unparsed.tags)
|
|
880
|
+
tags = list(set([*unparsed_tags, *config_tags]))
|
|
881
|
+
tags.sort()
|
|
882
|
+
|
|
883
|
+
parsed = SavedQuery(
|
|
884
|
+
description=unparsed.description,
|
|
885
|
+
label=unparsed.label,
|
|
886
|
+
fqn=fqn,
|
|
887
|
+
name=unparsed.name,
|
|
888
|
+
original_file_path=self.yaml.path.original_file_path,
|
|
889
|
+
package_name=package_name,
|
|
890
|
+
path=path,
|
|
891
|
+
resource_type=NodeType.SavedQuery,
|
|
892
|
+
unique_id=unique_id,
|
|
893
|
+
query_params=self._get_query_params(unparsed.query_params),
|
|
894
|
+
exports=[self._get_export(export, config) for export in unparsed.exports],
|
|
895
|
+
config=config,
|
|
896
|
+
unrendered_config=unrendered_config,
|
|
897
|
+
group=config.group,
|
|
898
|
+
tags=tags,
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
for export in parsed.exports:
|
|
902
|
+
self.schema_parser.update_parsed_node_relation_names(export, export.config.to_dict()) # type: ignore
|
|
903
|
+
|
|
904
|
+
if not export.config.schema_name:
|
|
905
|
+
export.config.schema_name = getattr(export, "schema", None)
|
|
906
|
+
delattr(export, "schema")
|
|
907
|
+
|
|
908
|
+
export.config.database = getattr(export, "database", None) or export.config.database
|
|
909
|
+
delattr(export, "database")
|
|
910
|
+
|
|
911
|
+
if not export.config.alias:
|
|
912
|
+
export.config.alias = getattr(export, "alias", None)
|
|
913
|
+
delattr(export, "alias")
|
|
914
|
+
|
|
915
|
+
delattr(export, "relation_name")
|
|
916
|
+
|
|
917
|
+
# Only add thes saved query if it's enabled, otherwise we track it with other diabled nodes
|
|
918
|
+
assert isinstance(self.yaml.file, SchemaSourceFile)
|
|
919
|
+
if parsed.config.enabled:
|
|
920
|
+
self.manifest.add_saved_query(self.yaml.file, parsed)
|
|
921
|
+
else:
|
|
922
|
+
self.manifest.add_disabled(self.yaml.file, parsed)
|
|
923
|
+
|
|
924
|
+
def parse(self) -> ParseResult:
|
|
925
|
+
for data in self.get_key_dicts():
|
|
926
|
+
try:
|
|
927
|
+
UnparsedSavedQuery.validate(data)
|
|
928
|
+
unparsed = UnparsedSavedQuery.from_dict(data)
|
|
929
|
+
except (ValidationError, JSONValidationError) as exc:
|
|
930
|
+
raise YamlParseDictError(self.yaml.path, self.key, data, exc)
|
|
931
|
+
|
|
932
|
+
self.parse_saved_query(unparsed)
|
|
933
|
+
|
|
934
|
+
# The supertype (YamlReader) requires `parse` to return a ParseResult, so
|
|
935
|
+
# we return an empty one because we don't have one to actually return.
|
|
936
|
+
return ParseResult()
|