dvt-core 1.11.0b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dvt-core might be problematic. Click here for more details.
- dvt/__init__.py +7 -0
- dvt/_pydantic_shim.py +26 -0
- dvt/adapters/__init__.py +16 -0
- dvt/adapters/multi_adapter_manager.py +268 -0
- dvt/artifacts/__init__.py +0 -0
- dvt/artifacts/exceptions/__init__.py +1 -0
- dvt/artifacts/exceptions/schemas.py +31 -0
- dvt/artifacts/resources/__init__.py +116 -0
- dvt/artifacts/resources/base.py +68 -0
- dvt/artifacts/resources/types.py +93 -0
- dvt/artifacts/resources/v1/analysis.py +10 -0
- dvt/artifacts/resources/v1/catalog.py +23 -0
- dvt/artifacts/resources/v1/components.py +275 -0
- dvt/artifacts/resources/v1/config.py +282 -0
- dvt/artifacts/resources/v1/documentation.py +11 -0
- dvt/artifacts/resources/v1/exposure.py +52 -0
- dvt/artifacts/resources/v1/function.py +53 -0
- dvt/artifacts/resources/v1/generic_test.py +32 -0
- dvt/artifacts/resources/v1/group.py +22 -0
- dvt/artifacts/resources/v1/hook.py +11 -0
- dvt/artifacts/resources/v1/macro.py +30 -0
- dvt/artifacts/resources/v1/metric.py +173 -0
- dvt/artifacts/resources/v1/model.py +146 -0
- dvt/artifacts/resources/v1/owner.py +10 -0
- dvt/artifacts/resources/v1/saved_query.py +112 -0
- dvt/artifacts/resources/v1/seed.py +42 -0
- dvt/artifacts/resources/v1/semantic_layer_components.py +72 -0
- dvt/artifacts/resources/v1/semantic_model.py +315 -0
- dvt/artifacts/resources/v1/singular_test.py +14 -0
- dvt/artifacts/resources/v1/snapshot.py +92 -0
- dvt/artifacts/resources/v1/source_definition.py +85 -0
- dvt/artifacts/resources/v1/sql_operation.py +10 -0
- dvt/artifacts/resources/v1/unit_test_definition.py +78 -0
- dvt/artifacts/schemas/__init__.py +0 -0
- dvt/artifacts/schemas/base.py +191 -0
- dvt/artifacts/schemas/batch_results.py +24 -0
- dvt/artifacts/schemas/catalog/__init__.py +12 -0
- dvt/artifacts/schemas/catalog/v1/__init__.py +0 -0
- dvt/artifacts/schemas/catalog/v1/catalog.py +60 -0
- dvt/artifacts/schemas/freshness/__init__.py +1 -0
- dvt/artifacts/schemas/freshness/v3/__init__.py +0 -0
- dvt/artifacts/schemas/freshness/v3/freshness.py +159 -0
- dvt/artifacts/schemas/manifest/__init__.py +2 -0
- dvt/artifacts/schemas/manifest/v12/__init__.py +0 -0
- dvt/artifacts/schemas/manifest/v12/manifest.py +212 -0
- dvt/artifacts/schemas/results.py +148 -0
- dvt/artifacts/schemas/run/__init__.py +2 -0
- dvt/artifacts/schemas/run/v5/__init__.py +0 -0
- dvt/artifacts/schemas/run/v5/run.py +184 -0
- dvt/artifacts/schemas/upgrades/__init__.py +4 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
- dvt/artifacts/utils/validation.py +153 -0
- dvt/cli/__init__.py +1 -0
- dvt/cli/context.py +16 -0
- dvt/cli/exceptions.py +56 -0
- dvt/cli/flags.py +558 -0
- dvt/cli/main.py +971 -0
- dvt/cli/option_types.py +121 -0
- dvt/cli/options.py +79 -0
- dvt/cli/params.py +803 -0
- dvt/cli/requires.py +478 -0
- dvt/cli/resolvers.py +32 -0
- dvt/cli/types.py +40 -0
- dvt/clients/__init__.py +0 -0
- dvt/clients/checked_load.py +82 -0
- dvt/clients/git.py +164 -0
- dvt/clients/jinja.py +206 -0
- dvt/clients/jinja_static.py +245 -0
- dvt/clients/registry.py +192 -0
- dvt/clients/yaml_helper.py +68 -0
- dvt/compilation.py +833 -0
- dvt/compute/__init__.py +26 -0
- dvt/compute/base.py +288 -0
- dvt/compute/engines/__init__.py +13 -0
- dvt/compute/engines/duckdb_engine.py +368 -0
- dvt/compute/engines/spark_engine.py +273 -0
- dvt/compute/query_analyzer.py +212 -0
- dvt/compute/router.py +483 -0
- dvt/config/__init__.py +4 -0
- dvt/config/catalogs.py +95 -0
- dvt/config/compute_config.py +406 -0
- dvt/config/profile.py +411 -0
- dvt/config/profiles_v2.py +464 -0
- dvt/config/project.py +893 -0
- dvt/config/renderer.py +232 -0
- dvt/config/runtime.py +491 -0
- dvt/config/selectors.py +209 -0
- dvt/config/utils.py +78 -0
- dvt/connectors/.gitignore +6 -0
- dvt/connectors/README.md +306 -0
- dvt/connectors/catalog.yml +217 -0
- dvt/connectors/download_connectors.py +300 -0
- dvt/constants.py +29 -0
- dvt/context/__init__.py +0 -0
- dvt/context/base.py +746 -0
- dvt/context/configured.py +136 -0
- dvt/context/context_config.py +350 -0
- dvt/context/docs.py +82 -0
- dvt/context/exceptions_jinja.py +179 -0
- dvt/context/macro_resolver.py +195 -0
- dvt/context/macros.py +171 -0
- dvt/context/manifest.py +73 -0
- dvt/context/providers.py +2198 -0
- dvt/context/query_header.py +14 -0
- dvt/context/secret.py +59 -0
- dvt/context/target.py +74 -0
- dvt/contracts/__init__.py +0 -0
- dvt/contracts/files.py +413 -0
- dvt/contracts/graph/__init__.py +0 -0
- dvt/contracts/graph/manifest.py +1904 -0
- dvt/contracts/graph/metrics.py +98 -0
- dvt/contracts/graph/model_config.py +71 -0
- dvt/contracts/graph/node_args.py +42 -0
- dvt/contracts/graph/nodes.py +1806 -0
- dvt/contracts/graph/semantic_manifest.py +233 -0
- dvt/contracts/graph/unparsed.py +812 -0
- dvt/contracts/project.py +417 -0
- dvt/contracts/results.py +53 -0
- dvt/contracts/selection.py +23 -0
- dvt/contracts/sql.py +86 -0
- dvt/contracts/state.py +69 -0
- dvt/contracts/util.py +46 -0
- dvt/deprecations.py +347 -0
- dvt/deps/__init__.py +0 -0
- dvt/deps/base.py +153 -0
- dvt/deps/git.py +196 -0
- dvt/deps/local.py +80 -0
- dvt/deps/registry.py +131 -0
- dvt/deps/resolver.py +149 -0
- dvt/deps/tarball.py +121 -0
- dvt/docs/source/_ext/dbt_click.py +118 -0
- dvt/docs/source/conf.py +32 -0
- dvt/env_vars.py +64 -0
- dvt/event_time/event_time.py +40 -0
- dvt/event_time/sample_window.py +60 -0
- dvt/events/__init__.py +16 -0
- dvt/events/base_types.py +37 -0
- dvt/events/core_types_pb2.py +2 -0
- dvt/events/logging.py +109 -0
- dvt/events/types.py +2534 -0
- dvt/exceptions.py +1487 -0
- dvt/flags.py +89 -0
- dvt/graph/__init__.py +11 -0
- dvt/graph/cli.py +248 -0
- dvt/graph/graph.py +172 -0
- dvt/graph/queue.py +213 -0
- dvt/graph/selector.py +375 -0
- dvt/graph/selector_methods.py +976 -0
- dvt/graph/selector_spec.py +223 -0
- dvt/graph/thread_pool.py +18 -0
- dvt/hooks.py +21 -0
- dvt/include/README.md +49 -0
- dvt/include/__init__.py +3 -0
- dvt/include/global_project.py +4 -0
- dvt/include/starter_project/.gitignore +4 -0
- dvt/include/starter_project/README.md +15 -0
- dvt/include/starter_project/__init__.py +3 -0
- dvt/include/starter_project/analyses/.gitkeep +0 -0
- dvt/include/starter_project/dvt_project.yml +36 -0
- dvt/include/starter_project/macros/.gitkeep +0 -0
- dvt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
- dvt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
- dvt/include/starter_project/models/example/schema.yml +21 -0
- dvt/include/starter_project/seeds/.gitkeep +0 -0
- dvt/include/starter_project/snapshots/.gitkeep +0 -0
- dvt/include/starter_project/tests/.gitkeep +0 -0
- dvt/internal_deprecations.py +27 -0
- dvt/jsonschemas/__init__.py +3 -0
- dvt/jsonschemas/jsonschemas.py +309 -0
- dvt/jsonschemas/project/0.0.110.json +4717 -0
- dvt/jsonschemas/project/0.0.85.json +2015 -0
- dvt/jsonschemas/resources/0.0.110.json +2636 -0
- dvt/jsonschemas/resources/0.0.85.json +2536 -0
- dvt/jsonschemas/resources/latest.json +6773 -0
- dvt/links.py +4 -0
- dvt/materializations/__init__.py +0 -0
- dvt/materializations/incremental/__init__.py +0 -0
- dvt/materializations/incremental/microbatch.py +235 -0
- dvt/mp_context.py +8 -0
- dvt/node_types.py +37 -0
- dvt/parser/__init__.py +23 -0
- dvt/parser/analysis.py +21 -0
- dvt/parser/base.py +549 -0
- dvt/parser/common.py +267 -0
- dvt/parser/docs.py +52 -0
- dvt/parser/fixtures.py +51 -0
- dvt/parser/functions.py +30 -0
- dvt/parser/generic_test.py +100 -0
- dvt/parser/generic_test_builders.py +334 -0
- dvt/parser/hooks.py +119 -0
- dvt/parser/macros.py +137 -0
- dvt/parser/manifest.py +2204 -0
- dvt/parser/models.py +574 -0
- dvt/parser/partial.py +1179 -0
- dvt/parser/read_files.py +445 -0
- dvt/parser/schema_generic_tests.py +423 -0
- dvt/parser/schema_renderer.py +111 -0
- dvt/parser/schema_yaml_readers.py +936 -0
- dvt/parser/schemas.py +1467 -0
- dvt/parser/search.py +149 -0
- dvt/parser/seeds.py +28 -0
- dvt/parser/singular_test.py +20 -0
- dvt/parser/snapshots.py +44 -0
- dvt/parser/sources.py +557 -0
- dvt/parser/sql.py +63 -0
- dvt/parser/unit_tests.py +622 -0
- dvt/plugins/__init__.py +20 -0
- dvt/plugins/contracts.py +10 -0
- dvt/plugins/exceptions.py +2 -0
- dvt/plugins/manager.py +164 -0
- dvt/plugins/manifest.py +21 -0
- dvt/profiler.py +20 -0
- dvt/py.typed +1 -0
- dvt/runners/__init__.py +2 -0
- dvt/runners/exposure_runner.py +7 -0
- dvt/runners/no_op_runner.py +46 -0
- dvt/runners/saved_query_runner.py +7 -0
- dvt/selected_resources.py +8 -0
- dvt/task/__init__.py +0 -0
- dvt/task/base.py +504 -0
- dvt/task/build.py +197 -0
- dvt/task/clean.py +57 -0
- dvt/task/clone.py +162 -0
- dvt/task/compile.py +151 -0
- dvt/task/compute.py +366 -0
- dvt/task/debug.py +650 -0
- dvt/task/deps.py +280 -0
- dvt/task/docs/__init__.py +3 -0
- dvt/task/docs/generate.py +408 -0
- dvt/task/docs/index.html +250 -0
- dvt/task/docs/serve.py +28 -0
- dvt/task/freshness.py +323 -0
- dvt/task/function.py +122 -0
- dvt/task/group_lookup.py +46 -0
- dvt/task/init.py +374 -0
- dvt/task/list.py +237 -0
- dvt/task/printer.py +176 -0
- dvt/task/profiles.py +256 -0
- dvt/task/retry.py +175 -0
- dvt/task/run.py +1146 -0
- dvt/task/run_operation.py +142 -0
- dvt/task/runnable.py +802 -0
- dvt/task/seed.py +104 -0
- dvt/task/show.py +150 -0
- dvt/task/snapshot.py +57 -0
- dvt/task/sql.py +111 -0
- dvt/task/test.py +464 -0
- dvt/tests/fixtures/__init__.py +1 -0
- dvt/tests/fixtures/project.py +620 -0
- dvt/tests/util.py +651 -0
- dvt/tracking.py +529 -0
- dvt/utils/__init__.py +3 -0
- dvt/utils/artifact_upload.py +151 -0
- dvt/utils/utils.py +408 -0
- dvt/version.py +249 -0
- dvt_core-1.11.0b4.dist-info/METADATA +252 -0
- dvt_core-1.11.0b4.dist-info/RECORD +261 -0
- dvt_core-1.11.0b4.dist-info/WHEEL +5 -0
- dvt_core-1.11.0b4.dist-info/entry_points.txt +2 -0
- dvt_core-1.11.0b4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,976 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from fnmatch import fnmatch
|
|
3
|
+
from itertools import chain
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import (
|
|
6
|
+
Any,
|
|
7
|
+
Callable,
|
|
8
|
+
Dict,
|
|
9
|
+
Iterator,
|
|
10
|
+
List,
|
|
11
|
+
Optional,
|
|
12
|
+
Set,
|
|
13
|
+
Tuple,
|
|
14
|
+
Type,
|
|
15
|
+
Union,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
from dvt.contracts.graph.manifest import Manifest
|
|
19
|
+
from dvt.contracts.graph.nodes import (
|
|
20
|
+
Exposure,
|
|
21
|
+
FunctionNode,
|
|
22
|
+
GenericTestNode,
|
|
23
|
+
ManifestNode,
|
|
24
|
+
Metric,
|
|
25
|
+
ModelNode,
|
|
26
|
+
ResultNode,
|
|
27
|
+
SavedQuery,
|
|
28
|
+
SemanticModel,
|
|
29
|
+
SingularTestNode,
|
|
30
|
+
SourceDefinition,
|
|
31
|
+
UnitTestDefinition,
|
|
32
|
+
)
|
|
33
|
+
from dvt.contracts.graph.unparsed import UnparsedVersion
|
|
34
|
+
from dvt.contracts.state import PreviousState
|
|
35
|
+
from dvt.node_types import NodeType
|
|
36
|
+
|
|
37
|
+
from dbt_common.dataclass_schema import StrEnum
|
|
38
|
+
from dbt_common.events.contextvars import get_project_root
|
|
39
|
+
from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
|
|
40
|
+
|
|
41
|
+
from .graph import UniqueId
|
|
42
|
+
|
|
43
|
+
SELECTOR_GLOB = "*"
|
|
44
|
+
SELECTOR_DELIMITER = ":"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class MethodName(StrEnum):
|
|
48
|
+
FQN = "fqn"
|
|
49
|
+
Tag = "tag"
|
|
50
|
+
Group = "group"
|
|
51
|
+
Access = "access"
|
|
52
|
+
Source = "source"
|
|
53
|
+
Path = "path"
|
|
54
|
+
File = "file"
|
|
55
|
+
Package = "package"
|
|
56
|
+
Config = "config"
|
|
57
|
+
TestName = "test_name"
|
|
58
|
+
TestType = "test_type"
|
|
59
|
+
ResourceType = "resource_type"
|
|
60
|
+
State = "state"
|
|
61
|
+
Exposure = "exposure"
|
|
62
|
+
Metric = "metric"
|
|
63
|
+
Result = "result"
|
|
64
|
+
SourceStatus = "source_status"
|
|
65
|
+
Version = "version"
|
|
66
|
+
SemanticModel = "semantic_model"
|
|
67
|
+
SavedQuery = "saved_query"
|
|
68
|
+
UnitTest = "unit_test"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def is_selected_node(fqn: List[str], node_selector: str, is_versioned: bool) -> bool:
|
|
72
|
+
# If qualified_name exactly matches model name (fqn's leaf), return True
|
|
73
|
+
if is_versioned:
|
|
74
|
+
flat_node_selector = node_selector.split(".")
|
|
75
|
+
if fqn[-2] == node_selector:
|
|
76
|
+
return True
|
|
77
|
+
# If this is a versioned model, then the last two segments should be allowed to exactly match on either the '.' or '_' delimiter
|
|
78
|
+
elif "_".join(fqn[-2:]) == "_".join(flat_node_selector[-2:]):
|
|
79
|
+
return True
|
|
80
|
+
else:
|
|
81
|
+
if fqn[-1] == node_selector:
|
|
82
|
+
return True
|
|
83
|
+
# Flatten node parts. Dots in model names act as namespace separators
|
|
84
|
+
flat_fqn = [item for segment in fqn for item in segment.split(".")]
|
|
85
|
+
# Selector components cannot be more than fqn's
|
|
86
|
+
if len(flat_fqn) < len(node_selector.split(".")):
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
slurp_from_ix: Optional[int] = None
|
|
90
|
+
for i, selector_part in enumerate(node_selector.split(".")):
|
|
91
|
+
if any(wildcard in selector_part for wildcard in ("*", "?", "[", "]")):
|
|
92
|
+
slurp_from_ix = i
|
|
93
|
+
break
|
|
94
|
+
elif flat_fqn[i] == selector_part:
|
|
95
|
+
continue
|
|
96
|
+
else:
|
|
97
|
+
return False
|
|
98
|
+
|
|
99
|
+
if slurp_from_ix is not None:
|
|
100
|
+
# If we have a wildcard, we need to make sure that the selector matches the
|
|
101
|
+
# rest of the fqn, this is 100% backwards compatible with the old behavior of
|
|
102
|
+
# encountering a wildcard but more expressive in naturally allowing you to
|
|
103
|
+
# match the rest of the fqn with more advanced patterns
|
|
104
|
+
return fnmatch(
|
|
105
|
+
".".join(flat_fqn[slurp_from_ix:]),
|
|
106
|
+
".".join(node_selector.split(".")[slurp_from_ix:]),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# if we get all the way down here, then the node is a match
|
|
110
|
+
return True
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
SelectorTarget = Union[
|
|
114
|
+
SourceDefinition, ManifestNode, Exposure, Metric, SemanticModel, UnitTestDefinition, SavedQuery
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class SelectorMethod(metaclass=abc.ABCMeta):
|
|
119
|
+
def __init__(
|
|
120
|
+
self, manifest: Manifest, previous_state: Optional[PreviousState], arguments: List[str]
|
|
121
|
+
) -> None:
|
|
122
|
+
self.manifest: Manifest = manifest
|
|
123
|
+
self.previous_state = previous_state
|
|
124
|
+
self.arguments: List[str] = arguments
|
|
125
|
+
|
|
126
|
+
def parsed_nodes(
|
|
127
|
+
self, included_nodes: Set[UniqueId]
|
|
128
|
+
) -> Iterator[Tuple[UniqueId, ManifestNode]]:
|
|
129
|
+
|
|
130
|
+
for key, node in self.manifest.nodes.items():
|
|
131
|
+
unique_id = UniqueId(key)
|
|
132
|
+
if unique_id not in included_nodes:
|
|
133
|
+
continue
|
|
134
|
+
yield unique_id, node
|
|
135
|
+
|
|
136
|
+
def source_nodes(
|
|
137
|
+
self, included_nodes: Set[UniqueId]
|
|
138
|
+
) -> Iterator[Tuple[UniqueId, SourceDefinition]]:
|
|
139
|
+
|
|
140
|
+
for key, source in self.manifest.sources.items():
|
|
141
|
+
unique_id = UniqueId(key)
|
|
142
|
+
if unique_id not in included_nodes:
|
|
143
|
+
continue
|
|
144
|
+
yield unique_id, source
|
|
145
|
+
|
|
146
|
+
def exposure_nodes(self, included_nodes: Set[UniqueId]) -> Iterator[Tuple[UniqueId, Exposure]]:
|
|
147
|
+
|
|
148
|
+
for key, exposure in self.manifest.exposures.items():
|
|
149
|
+
unique_id = UniqueId(key)
|
|
150
|
+
if unique_id not in included_nodes:
|
|
151
|
+
continue
|
|
152
|
+
yield unique_id, exposure
|
|
153
|
+
|
|
154
|
+
def metric_nodes(self, included_nodes: Set[UniqueId]) -> Iterator[Tuple[UniqueId, Metric]]:
|
|
155
|
+
|
|
156
|
+
for key, metric in self.manifest.metrics.items():
|
|
157
|
+
unique_id = UniqueId(key)
|
|
158
|
+
if unique_id not in included_nodes:
|
|
159
|
+
continue
|
|
160
|
+
yield unique_id, metric
|
|
161
|
+
|
|
162
|
+
def unit_tests(
|
|
163
|
+
self, included_nodes: Set[UniqueId]
|
|
164
|
+
) -> Iterator[Tuple[UniqueId, UnitTestDefinition]]:
|
|
165
|
+
for unique_id, unit_test in self.manifest.unit_tests.items():
|
|
166
|
+
unique_id = UniqueId(unique_id)
|
|
167
|
+
if unique_id not in included_nodes:
|
|
168
|
+
continue
|
|
169
|
+
yield unique_id, unit_test
|
|
170
|
+
|
|
171
|
+
def parsed_and_unit_nodes(self, included_nodes: Set[UniqueId]):
|
|
172
|
+
yield from chain(
|
|
173
|
+
self.parsed_nodes(included_nodes),
|
|
174
|
+
self.unit_tests(included_nodes),
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
def semantic_model_nodes(
|
|
178
|
+
self, included_nodes: Set[UniqueId]
|
|
179
|
+
) -> Iterator[Tuple[UniqueId, SemanticModel]]:
|
|
180
|
+
|
|
181
|
+
for key, semantic_model in self.manifest.semantic_models.items():
|
|
182
|
+
unique_id = UniqueId(key)
|
|
183
|
+
if unique_id not in included_nodes:
|
|
184
|
+
continue
|
|
185
|
+
yield unique_id, semantic_model
|
|
186
|
+
|
|
187
|
+
def saved_query_nodes(
|
|
188
|
+
self, included_nodes: Set[UniqueId]
|
|
189
|
+
) -> Iterator[Tuple[UniqueId, SavedQuery]]:
|
|
190
|
+
|
|
191
|
+
for key, saved_query in self.manifest.saved_queries.items():
|
|
192
|
+
unique_id = UniqueId(key)
|
|
193
|
+
if unique_id not in included_nodes:
|
|
194
|
+
continue
|
|
195
|
+
yield unique_id, saved_query
|
|
196
|
+
|
|
197
|
+
def function_nodes(
|
|
198
|
+
self, included_nodes: Set[UniqueId]
|
|
199
|
+
) -> Iterator[Tuple[UniqueId, FunctionNode]]:
|
|
200
|
+
for key, function in self.manifest.functions.items():
|
|
201
|
+
unique_id = UniqueId(key)
|
|
202
|
+
if unique_id not in included_nodes:
|
|
203
|
+
continue
|
|
204
|
+
yield unique_id, function
|
|
205
|
+
|
|
206
|
+
def all_nodes(
|
|
207
|
+
self, included_nodes: Set[UniqueId]
|
|
208
|
+
) -> Iterator[Tuple[UniqueId, SelectorTarget]]:
|
|
209
|
+
yield from chain(
|
|
210
|
+
self.parsed_nodes(included_nodes),
|
|
211
|
+
self.source_nodes(included_nodes),
|
|
212
|
+
self.exposure_nodes(included_nodes),
|
|
213
|
+
self.metric_nodes(included_nodes),
|
|
214
|
+
self.unit_tests(included_nodes),
|
|
215
|
+
self.semantic_model_nodes(included_nodes),
|
|
216
|
+
self.saved_query_nodes(included_nodes),
|
|
217
|
+
self.function_nodes(included_nodes),
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
def configurable_nodes(
|
|
221
|
+
self, included_nodes: Set[UniqueId]
|
|
222
|
+
) -> Iterator[Tuple[UniqueId, ResultNode]]:
|
|
223
|
+
yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes))
|
|
224
|
+
|
|
225
|
+
def non_source_nodes(
|
|
226
|
+
self,
|
|
227
|
+
included_nodes: Set[UniqueId],
|
|
228
|
+
) -> Iterator[Tuple[UniqueId, Union[Exposure, ManifestNode, Metric]]]:
|
|
229
|
+
yield from chain(
|
|
230
|
+
self.parsed_nodes(included_nodes),
|
|
231
|
+
self.exposure_nodes(included_nodes),
|
|
232
|
+
self.metric_nodes(included_nodes),
|
|
233
|
+
self.unit_tests(included_nodes),
|
|
234
|
+
self.semantic_model_nodes(included_nodes),
|
|
235
|
+
self.saved_query_nodes(included_nodes),
|
|
236
|
+
self.function_nodes(included_nodes),
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
def groupable_nodes(
|
|
240
|
+
self,
|
|
241
|
+
included_nodes: Set[UniqueId],
|
|
242
|
+
) -> Iterator[Tuple[UniqueId, Union[ManifestNode, Metric]]]:
|
|
243
|
+
yield from chain(
|
|
244
|
+
self.parsed_nodes(included_nodes),
|
|
245
|
+
self.metric_nodes(included_nodes),
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
@abc.abstractmethod
|
|
249
|
+
def search(
|
|
250
|
+
self,
|
|
251
|
+
included_nodes: Set[UniqueId],
|
|
252
|
+
selector: str,
|
|
253
|
+
) -> Iterator[UniqueId]:
|
|
254
|
+
raise NotImplementedError("subclasses should implement this")
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class QualifiedNameSelectorMethod(SelectorMethod):
|
|
258
|
+
def node_is_match(self, qualified_name: str, fqn: List[str], is_versioned: bool) -> bool:
|
|
259
|
+
"""Determine if a qualified name matches an fqn for all package
|
|
260
|
+
names in the graph.
|
|
261
|
+
|
|
262
|
+
:param str qualified_name: The qualified name to match the nodes with
|
|
263
|
+
:param List[str] fqn: The node's fully qualified name in the graph.
|
|
264
|
+
"""
|
|
265
|
+
unscoped_fqn = fqn[1:]
|
|
266
|
+
|
|
267
|
+
if is_selected_node(fqn, qualified_name, is_versioned):
|
|
268
|
+
return True
|
|
269
|
+
# Match nodes across different packages
|
|
270
|
+
elif is_selected_node(unscoped_fqn, qualified_name, is_versioned):
|
|
271
|
+
return True
|
|
272
|
+
|
|
273
|
+
return False
|
|
274
|
+
|
|
275
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
276
|
+
"""Yield all nodes in the graph that match the selector.
|
|
277
|
+
|
|
278
|
+
:param str selector: The selector or node name
|
|
279
|
+
"""
|
|
280
|
+
non_source_nodes = list(self.non_source_nodes(included_nodes))
|
|
281
|
+
for unique_id, node in non_source_nodes:
|
|
282
|
+
if self.node_is_match(selector, node.fqn, node.is_versioned):
|
|
283
|
+
yield unique_id
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class TagSelectorMethod(SelectorMethod):
|
|
287
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
288
|
+
"""yields nodes from included that have the specified tag"""
|
|
289
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
290
|
+
if hasattr(node, "tags") and any(fnmatch(tag, selector) for tag in node.tags):
|
|
291
|
+
yield unique_id
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class GroupSelectorMethod(SelectorMethod):
|
|
295
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
296
|
+
"""yields nodes from included in the specified group"""
|
|
297
|
+
for unique_id, node in self.groupable_nodes(included_nodes):
|
|
298
|
+
node_group = node.config.get("group")
|
|
299
|
+
if node_group and fnmatch(node_group, selector):
|
|
300
|
+
yield unique_id
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class AccessSelectorMethod(SelectorMethod):
|
|
304
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
305
|
+
"""yields model nodes matching the specified access level"""
|
|
306
|
+
for unique_id, node in self.parsed_nodes(included_nodes):
|
|
307
|
+
if not isinstance(node, ModelNode):
|
|
308
|
+
continue
|
|
309
|
+
if selector == node.access:
|
|
310
|
+
yield unique_id
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
class SourceSelectorMethod(SelectorMethod):
|
|
314
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
315
|
+
"""yields nodes from included are the specified source."""
|
|
316
|
+
parts = selector.split(".")
|
|
317
|
+
target_package = SELECTOR_GLOB
|
|
318
|
+
if len(parts) == 1:
|
|
319
|
+
target_source, target_table = parts[0], SELECTOR_GLOB
|
|
320
|
+
elif len(parts) == 2:
|
|
321
|
+
target_source, target_table = parts
|
|
322
|
+
elif len(parts) == 3:
|
|
323
|
+
target_package, target_source, target_table = parts
|
|
324
|
+
else: # len(parts) > 3 or len(parts) == 0
|
|
325
|
+
msg = (
|
|
326
|
+
'Invalid source selector value "{}". Sources must be of the '
|
|
327
|
+
"form `${{source_name}}`, "
|
|
328
|
+
"`${{source_name}}.${{target_name}}`, or "
|
|
329
|
+
"`${{package_name}}.${{source_name}}.${{target_name}}"
|
|
330
|
+
).format(selector)
|
|
331
|
+
raise DbtRuntimeError(msg)
|
|
332
|
+
|
|
333
|
+
for unique_id, node in self.source_nodes(included_nodes):
|
|
334
|
+
if not fnmatch(node.package_name, target_package):
|
|
335
|
+
continue
|
|
336
|
+
if not fnmatch(node.source_name, target_source):
|
|
337
|
+
continue
|
|
338
|
+
if not fnmatch(node.name, target_table):
|
|
339
|
+
continue
|
|
340
|
+
yield unique_id
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class ExposureSelectorMethod(SelectorMethod):
|
|
344
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
345
|
+
parts = selector.split(".")
|
|
346
|
+
target_package = SELECTOR_GLOB
|
|
347
|
+
if len(parts) == 1:
|
|
348
|
+
target_name = parts[0]
|
|
349
|
+
elif len(parts) == 2:
|
|
350
|
+
target_package, target_name = parts
|
|
351
|
+
else:
|
|
352
|
+
msg = (
|
|
353
|
+
'Invalid exposure selector value "{}". Exposures must be of '
|
|
354
|
+
"the form ${{exposure_name}} or "
|
|
355
|
+
"${{exposure_package.exposure_name}}"
|
|
356
|
+
).format(selector)
|
|
357
|
+
raise DbtRuntimeError(msg)
|
|
358
|
+
|
|
359
|
+
for unique_id, node in self.exposure_nodes(included_nodes):
|
|
360
|
+
if not fnmatch(node.package_name, target_package):
|
|
361
|
+
continue
|
|
362
|
+
if not fnmatch(node.name, target_name):
|
|
363
|
+
continue
|
|
364
|
+
|
|
365
|
+
yield unique_id
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
class MetricSelectorMethod(SelectorMethod):
|
|
369
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
370
|
+
parts = selector.split(".")
|
|
371
|
+
target_package = SELECTOR_GLOB
|
|
372
|
+
if len(parts) == 1:
|
|
373
|
+
target_name = parts[0]
|
|
374
|
+
elif len(parts) == 2:
|
|
375
|
+
target_package, target_name = parts
|
|
376
|
+
else:
|
|
377
|
+
msg = (
|
|
378
|
+
'Invalid metric selector value "{}". Metrics must be of '
|
|
379
|
+
"the form ${{metric_name}} or "
|
|
380
|
+
"${{metric_package.metric_name}}"
|
|
381
|
+
).format(selector)
|
|
382
|
+
raise DbtRuntimeError(msg)
|
|
383
|
+
|
|
384
|
+
for unique_id, node in self.metric_nodes(included_nodes):
|
|
385
|
+
if not fnmatch(node.package_name, target_package):
|
|
386
|
+
continue
|
|
387
|
+
if not fnmatch(node.name, target_name):
|
|
388
|
+
continue
|
|
389
|
+
|
|
390
|
+
yield unique_id
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class SemanticModelSelectorMethod(SelectorMethod):
|
|
394
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
395
|
+
parts = selector.split(".")
|
|
396
|
+
target_package = SELECTOR_GLOB
|
|
397
|
+
if len(parts) == 1:
|
|
398
|
+
target_name = parts[0]
|
|
399
|
+
elif len(parts) == 2:
|
|
400
|
+
target_package, target_name = parts
|
|
401
|
+
else:
|
|
402
|
+
msg = (
|
|
403
|
+
'Invalid semantic model selector value "{}". Semantic models must be of '
|
|
404
|
+
"the form ${{semantic_model_name}} or "
|
|
405
|
+
"${{semantic_model_package.semantic_model_name}}"
|
|
406
|
+
).format(selector)
|
|
407
|
+
raise DbtRuntimeError(msg)
|
|
408
|
+
|
|
409
|
+
for unique_id, node in self.semantic_model_nodes(included_nodes):
|
|
410
|
+
if not fnmatch(node.package_name, target_package):
|
|
411
|
+
continue
|
|
412
|
+
if not fnmatch(node.name, target_name):
|
|
413
|
+
continue
|
|
414
|
+
|
|
415
|
+
yield unique_id
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
class SavedQuerySelectorMethod(SelectorMethod):
|
|
419
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
420
|
+
parts = selector.split(".")
|
|
421
|
+
target_package = SELECTOR_GLOB
|
|
422
|
+
if len(parts) == 1:
|
|
423
|
+
target_name = parts[0]
|
|
424
|
+
elif len(parts) == 2:
|
|
425
|
+
target_package, target_name = parts
|
|
426
|
+
else:
|
|
427
|
+
msg = (
|
|
428
|
+
'Invalid saved query selector value "{}". Saved queries must be of '
|
|
429
|
+
"the form ${{saved_query_name}} or "
|
|
430
|
+
"${{saved_query_package.saved_query_name}}"
|
|
431
|
+
).format(selector)
|
|
432
|
+
raise DbtRuntimeError(msg)
|
|
433
|
+
|
|
434
|
+
for unique_id, node in self.saved_query_nodes(included_nodes):
|
|
435
|
+
if not fnmatch(node.package_name, target_package):
|
|
436
|
+
continue
|
|
437
|
+
if not fnmatch(node.name, target_name):
|
|
438
|
+
continue
|
|
439
|
+
|
|
440
|
+
yield unique_id
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
class UnitTestSelectorMethod(SelectorMethod):
|
|
444
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
445
|
+
parts = selector.split(".")
|
|
446
|
+
target_package = SELECTOR_GLOB
|
|
447
|
+
if len(parts) == 1:
|
|
448
|
+
target_name = parts[0]
|
|
449
|
+
elif len(parts) == 2:
|
|
450
|
+
target_package, target_name = parts
|
|
451
|
+
else:
|
|
452
|
+
msg = (
|
|
453
|
+
'Invalid unit test selector value "{}". Saved queries must be of '
|
|
454
|
+
"the form ${{unit_test_name}} or "
|
|
455
|
+
"${{unit_test_package_name.unit_test_name}}"
|
|
456
|
+
).format(selector)
|
|
457
|
+
raise DbtRuntimeError(msg)
|
|
458
|
+
|
|
459
|
+
for unique_id, node in self.unit_tests(included_nodes):
|
|
460
|
+
if not fnmatch(node.package_name, target_package):
|
|
461
|
+
continue
|
|
462
|
+
if not fnmatch(node.name, target_name):
|
|
463
|
+
continue
|
|
464
|
+
|
|
465
|
+
yield unique_id
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
class PathSelectorMethod(SelectorMethod):
|
|
469
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
470
|
+
"""Yields nodes from included that match the given path."""
|
|
471
|
+
# get project root from contextvar
|
|
472
|
+
project_root = get_project_root()
|
|
473
|
+
if project_root:
|
|
474
|
+
root = Path(project_root)
|
|
475
|
+
else:
|
|
476
|
+
root = Path.cwd()
|
|
477
|
+
paths = set(p.relative_to(root) for p in root.glob(selector))
|
|
478
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
479
|
+
ofp = Path(node.original_file_path)
|
|
480
|
+
if ofp in paths:
|
|
481
|
+
yield unique_id
|
|
482
|
+
if hasattr(node, "patch_path") and node.patch_path: # type: ignore
|
|
483
|
+
pfp = node.patch_path.split("://")[1] # type: ignore
|
|
484
|
+
ymlfp = Path(pfp)
|
|
485
|
+
if ymlfp in paths:
|
|
486
|
+
yield unique_id
|
|
487
|
+
if any(parent in paths for parent in ofp.parents):
|
|
488
|
+
yield unique_id
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
class FileSelectorMethod(SelectorMethod):
|
|
492
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
493
|
+
"""Yields nodes from included that match the given file name."""
|
|
494
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
495
|
+
if fnmatch(Path(node.original_file_path).name, selector):
|
|
496
|
+
yield unique_id
|
|
497
|
+
elif fnmatch(Path(node.original_file_path).stem, selector):
|
|
498
|
+
yield unique_id
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
class PackageSelectorMethod(SelectorMethod):
|
|
502
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
503
|
+
"""Yields nodes from included that have the specified package"""
|
|
504
|
+
# `this` is an alias for the current dbt project name
|
|
505
|
+
if selector == "this" and self.manifest.metadata.project_name is not None:
|
|
506
|
+
selector = self.manifest.metadata.project_name
|
|
507
|
+
|
|
508
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
509
|
+
if fnmatch(node.package_name, selector):
|
|
510
|
+
yield unique_id
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _getattr_descend(obj: Any, attrs: List[str]) -> Any:
|
|
514
|
+
value = obj
|
|
515
|
+
for attr in attrs:
|
|
516
|
+
try:
|
|
517
|
+
value = getattr(value, attr)
|
|
518
|
+
except AttributeError:
|
|
519
|
+
# if it implements getitem (dict, list, ...), use that. On failure,
|
|
520
|
+
# raise an attribute error instead of the KeyError, TypeError, etc.
|
|
521
|
+
# that arbitrary getitem calls might raise
|
|
522
|
+
try:
|
|
523
|
+
value = value[attr]
|
|
524
|
+
except Exception as exc:
|
|
525
|
+
raise AttributeError(f"'{type(value)}' object has no attribute '{attr}'") from exc
|
|
526
|
+
return value
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
class CaseInsensitive(str):
|
|
530
|
+
def __eq__(self, other):
|
|
531
|
+
if isinstance(other, str):
|
|
532
|
+
return self.upper() == other.upper()
|
|
533
|
+
else:
|
|
534
|
+
return self.upper() == other
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class ConfigSelectorMethod(SelectorMethod):
|
|
538
|
+
def search(
|
|
539
|
+
self,
|
|
540
|
+
included_nodes: Set[UniqueId],
|
|
541
|
+
selector: Any,
|
|
542
|
+
) -> Iterator[UniqueId]:
|
|
543
|
+
parts = self.arguments
|
|
544
|
+
# special case: if the user wanted to compare test severity,
|
|
545
|
+
# make the comparison case-insensitive
|
|
546
|
+
if parts == ["severity"]:
|
|
547
|
+
selector = CaseInsensitive(selector)
|
|
548
|
+
|
|
549
|
+
# search sources is kind of useless now source configs only have
|
|
550
|
+
# 'enabled', which you can't really filter on anyway, but maybe we'll
|
|
551
|
+
# add more someday, so search them anyway.
|
|
552
|
+
for unique_id, node in self.configurable_nodes(included_nodes):
|
|
553
|
+
try:
|
|
554
|
+
value = _getattr_descend(node.config, parts)
|
|
555
|
+
except AttributeError:
|
|
556
|
+
continue
|
|
557
|
+
else:
|
|
558
|
+
if isinstance(value, list):
|
|
559
|
+
if (
|
|
560
|
+
(selector in value)
|
|
561
|
+
or (CaseInsensitive(selector) == "true" and True in value)
|
|
562
|
+
or (CaseInsensitive(selector) == "false" and False in value)
|
|
563
|
+
):
|
|
564
|
+
yield unique_id
|
|
565
|
+
else:
|
|
566
|
+
if (
|
|
567
|
+
(selector == value)
|
|
568
|
+
or (CaseInsensitive(selector) == "true" and value is True)
|
|
569
|
+
or (CaseInsensitive(selector) == "false")
|
|
570
|
+
and value is False
|
|
571
|
+
):
|
|
572
|
+
yield unique_id
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
class ResourceTypeSelectorMethod(SelectorMethod):
|
|
576
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
577
|
+
try:
|
|
578
|
+
resource_type = NodeType(selector)
|
|
579
|
+
except ValueError as exc:
|
|
580
|
+
raise DbtRuntimeError(f'Invalid resource_type selector "{selector}"') from exc
|
|
581
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
582
|
+
if node.resource_type == resource_type:
|
|
583
|
+
yield unique_id
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
class TestNameSelectorMethod(SelectorMethod):
|
|
587
|
+
__test__ = False
|
|
588
|
+
|
|
589
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
590
|
+
for unique_id, node in self.parsed_and_unit_nodes(included_nodes):
|
|
591
|
+
if node.resource_type == NodeType.Test and hasattr(node, "test_metadata"):
|
|
592
|
+
if fnmatch(node.test_metadata.name, selector): # type: ignore[union-attr]
|
|
593
|
+
yield unique_id
|
|
594
|
+
elif node.resource_type == NodeType.Unit:
|
|
595
|
+
if fnmatch(node.name, selector):
|
|
596
|
+
yield unique_id
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
class TestTypeSelectorMethod(SelectorMethod):
|
|
600
|
+
__test__ = False
|
|
601
|
+
|
|
602
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
603
|
+
search_types: List[Any]
|
|
604
|
+
# continue supporting 'schema' + 'data' for backwards compatibility
|
|
605
|
+
if selector in ("generic", "schema"):
|
|
606
|
+
search_types = [GenericTestNode]
|
|
607
|
+
elif selector in ("data"):
|
|
608
|
+
search_types = [GenericTestNode, SingularTestNode]
|
|
609
|
+
elif selector in ("singular"):
|
|
610
|
+
search_types = [SingularTestNode]
|
|
611
|
+
elif selector in ("unit"):
|
|
612
|
+
search_types = [UnitTestDefinition]
|
|
613
|
+
else:
|
|
614
|
+
raise DbtRuntimeError(
|
|
615
|
+
f'Invalid test type selector {selector}: expected "generic", "singular", "unit", or "data"'
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
for unique_id, node in self.parsed_and_unit_nodes(included_nodes):
|
|
619
|
+
if isinstance(node, tuple(search_types)):
|
|
620
|
+
yield unique_id
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
class StateSelectorMethod(SelectorMethod):
|
|
624
|
+
def __init__(self, *args, **kwargs) -> None:
|
|
625
|
+
super().__init__(*args, **kwargs)
|
|
626
|
+
self.modified_macros: Optional[List[str]] = None
|
|
627
|
+
|
|
628
|
+
def _macros_modified(self) -> List[str]:
|
|
629
|
+
# we checked in the caller!
|
|
630
|
+
if self.previous_state is None or self.previous_state.manifest is None:
|
|
631
|
+
raise DbtInternalError("No comparison manifest in _macros_modified")
|
|
632
|
+
old_macros = self.previous_state.manifest.macros
|
|
633
|
+
new_macros = self.manifest.macros
|
|
634
|
+
|
|
635
|
+
modified = []
|
|
636
|
+
for uid, macro in new_macros.items():
|
|
637
|
+
if uid in old_macros:
|
|
638
|
+
old_macro = old_macros[uid]
|
|
639
|
+
if macro.macro_sql != old_macro.macro_sql:
|
|
640
|
+
modified.append(uid)
|
|
641
|
+
else:
|
|
642
|
+
modified.append(uid)
|
|
643
|
+
|
|
644
|
+
for uid, _ in old_macros.items():
|
|
645
|
+
if uid not in new_macros:
|
|
646
|
+
modified.append(uid)
|
|
647
|
+
|
|
648
|
+
return modified
|
|
649
|
+
|
|
650
|
+
def recursively_check_macros_modified(self, node, visited_macros):
|
|
651
|
+
if not hasattr(node, "depends_on"):
|
|
652
|
+
return False
|
|
653
|
+
|
|
654
|
+
for macro_uid in node.depends_on.macros:
|
|
655
|
+
if macro_uid in visited_macros:
|
|
656
|
+
continue
|
|
657
|
+
visited_macros.append(macro_uid)
|
|
658
|
+
|
|
659
|
+
if macro_uid in self.modified_macros:
|
|
660
|
+
return True
|
|
661
|
+
|
|
662
|
+
# this macro hasn't been modified, but depends on other
|
|
663
|
+
# macros which each need to be tested for modification
|
|
664
|
+
macro_node = self.manifest.macros[macro_uid]
|
|
665
|
+
if len(macro_node.depends_on.macros) > 0:
|
|
666
|
+
upstream_macros_changed = self.recursively_check_macros_modified(
|
|
667
|
+
macro_node, visited_macros
|
|
668
|
+
)
|
|
669
|
+
if upstream_macros_changed:
|
|
670
|
+
return True
|
|
671
|
+
continue
|
|
672
|
+
|
|
673
|
+
# this macro hasn't been modified, but we haven't checked
|
|
674
|
+
# the other macros the node depends on, so keep looking
|
|
675
|
+
if len(node.depends_on.macros) > len(visited_macros):
|
|
676
|
+
continue
|
|
677
|
+
|
|
678
|
+
return False
|
|
679
|
+
|
|
680
|
+
def check_macros_modified(self, node):
|
|
681
|
+
# check if there are any changes in macros the first time
|
|
682
|
+
if self.modified_macros is None:
|
|
683
|
+
self.modified_macros = self._macros_modified()
|
|
684
|
+
# no macros have been modified, skip looping entirely
|
|
685
|
+
if not self.modified_macros:
|
|
686
|
+
return False
|
|
687
|
+
# recursively loop through upstream macros to see if any is modified
|
|
688
|
+
else:
|
|
689
|
+
visited_macros = []
|
|
690
|
+
return self.recursively_check_macros_modified(node, visited_macros)
|
|
691
|
+
|
|
692
|
+
# TODO check modifed_content and check_modified macro seems a bit redundent
|
|
693
|
+
def check_modified_content(
|
|
694
|
+
self, old: Optional[SelectorTarget], new: SelectorTarget, adapter_type: str
|
|
695
|
+
) -> bool:
|
|
696
|
+
different_contents = False
|
|
697
|
+
if isinstance(
|
|
698
|
+
new,
|
|
699
|
+
(SourceDefinition, Exposure, Metric, SemanticModel, UnitTestDefinition, SavedQuery),
|
|
700
|
+
):
|
|
701
|
+
# these all overwrite `same_contents`
|
|
702
|
+
different_contents = not new.same_contents(old) # type: ignore
|
|
703
|
+
elif new: # because we also pull in deleted/disabled nodes, this could be None
|
|
704
|
+
different_contents = not new.same_contents(old, adapter_type) # type: ignore
|
|
705
|
+
|
|
706
|
+
upstream_macro_change = self.check_macros_modified(new)
|
|
707
|
+
|
|
708
|
+
check_modified_contract = False
|
|
709
|
+
if isinstance(old, ModelNode):
|
|
710
|
+
func = self.check_modified_contract("same_contract", adapter_type)
|
|
711
|
+
check_modified_contract = func(old, new)
|
|
712
|
+
|
|
713
|
+
return different_contents or upstream_macro_change or check_modified_contract
|
|
714
|
+
|
|
715
|
+
def check_unmodified_content(
|
|
716
|
+
self, old: Optional[SelectorTarget], new: SelectorTarget, adapter_type: str
|
|
717
|
+
) -> bool:
|
|
718
|
+
return not self.check_modified_content(old, new, adapter_type)
|
|
719
|
+
|
|
720
|
+
def check_modified_macros(self, old, new: SelectorTarget) -> bool:
|
|
721
|
+
return self.check_macros_modified(new)
|
|
722
|
+
|
|
723
|
+
@staticmethod
|
|
724
|
+
def check_modified_factory(
|
|
725
|
+
compare_method: str,
|
|
726
|
+
) -> Callable[[Optional[SelectorTarget], SelectorTarget], bool]:
|
|
727
|
+
# get a function that compares two selector target based on compare method provided
|
|
728
|
+
def check_modified_things(old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
|
|
729
|
+
if hasattr(new, compare_method):
|
|
730
|
+
# when old body does not exist or old and new are not the same
|
|
731
|
+
return not old or not getattr(new, compare_method)(old) # type: ignore
|
|
732
|
+
else:
|
|
733
|
+
return False
|
|
734
|
+
|
|
735
|
+
return check_modified_things
|
|
736
|
+
|
|
737
|
+
@staticmethod
|
|
738
|
+
def check_modified_contract(
|
|
739
|
+
compare_method: str,
|
|
740
|
+
adapter_type: Optional[str],
|
|
741
|
+
) -> Callable[[Optional[SelectorTarget], SelectorTarget], bool]:
|
|
742
|
+
# get a function that compares two selector target based on compare method provided
|
|
743
|
+
def check_modified_contract(old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
|
|
744
|
+
if new is None and hasattr(old, compare_method + "_removed"):
|
|
745
|
+
return getattr(old, compare_method + "_removed")()
|
|
746
|
+
elif hasattr(new, compare_method):
|
|
747
|
+
# when old body does not exist or old and new are not the same
|
|
748
|
+
return not old or not getattr(new, compare_method)(old, adapter_type) # type: ignore
|
|
749
|
+
else:
|
|
750
|
+
return False
|
|
751
|
+
|
|
752
|
+
return check_modified_contract
|
|
753
|
+
|
|
754
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
755
|
+
if self.previous_state is None or self.previous_state.manifest is None:
|
|
756
|
+
raise DbtRuntimeError("Got a state selector method, but no comparison manifest")
|
|
757
|
+
|
|
758
|
+
adapter_type = self.manifest.metadata.adapter_type
|
|
759
|
+
|
|
760
|
+
state_checks = {
|
|
761
|
+
# it's new if there is no old version
|
|
762
|
+
"new": lambda old, new: old is None,
|
|
763
|
+
"old": lambda old, new: old is not None,
|
|
764
|
+
# use methods defined above to compare properties of old + new
|
|
765
|
+
"modified": self.check_modified_content,
|
|
766
|
+
"unmodified": self.check_unmodified_content,
|
|
767
|
+
"modified.body": self.check_modified_factory("same_body"),
|
|
768
|
+
"modified.configs": self.check_modified_factory("same_config"),
|
|
769
|
+
"modified.persisted_descriptions": self.check_modified_factory(
|
|
770
|
+
"same_persisted_description"
|
|
771
|
+
),
|
|
772
|
+
"modified.relation": self.check_modified_factory("same_database_representation"),
|
|
773
|
+
"modified.macros": self.check_modified_macros,
|
|
774
|
+
"modified.contract": self.check_modified_contract("same_contract", adapter_type),
|
|
775
|
+
}
|
|
776
|
+
if selector in state_checks:
|
|
777
|
+
checker = state_checks[selector]
|
|
778
|
+
else:
|
|
779
|
+
raise DbtRuntimeError(
|
|
780
|
+
f'Got an invalid selector "{selector}", expected one of ' f'"{list(state_checks)}"'
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
manifest: Manifest = self.previous_state.manifest
|
|
784
|
+
|
|
785
|
+
keyword_args = {} # initialize here to handle disabled node check below
|
|
786
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
787
|
+
previous_node: Optional[SelectorTarget] = None
|
|
788
|
+
|
|
789
|
+
if unique_id in manifest.nodes:
|
|
790
|
+
previous_node = manifest.nodes[unique_id]
|
|
791
|
+
elif unique_id in manifest.sources:
|
|
792
|
+
previous_node = SourceDefinition.from_resource(manifest.sources[unique_id])
|
|
793
|
+
elif unique_id in manifest.exposures:
|
|
794
|
+
previous_node = Exposure.from_resource(manifest.exposures[unique_id])
|
|
795
|
+
elif unique_id in manifest.metrics:
|
|
796
|
+
previous_node = Metric.from_resource(manifest.metrics[unique_id])
|
|
797
|
+
elif unique_id in manifest.semantic_models:
|
|
798
|
+
previous_node = SemanticModel.from_resource(manifest.semantic_models[unique_id])
|
|
799
|
+
elif unique_id in manifest.unit_tests:
|
|
800
|
+
previous_node = UnitTestDefinition.from_resource(manifest.unit_tests[unique_id])
|
|
801
|
+
elif unique_id in manifest.saved_queries:
|
|
802
|
+
previous_node = SavedQuery.from_resource(manifest.saved_queries[unique_id])
|
|
803
|
+
elif unique_id in manifest.functions:
|
|
804
|
+
previous_node = FunctionNode.from_resource(manifest.functions[unique_id])
|
|
805
|
+
|
|
806
|
+
if checker.__name__ in [
|
|
807
|
+
"same_contract",
|
|
808
|
+
"check_modified_content",
|
|
809
|
+
"check_unmodified_content",
|
|
810
|
+
]:
|
|
811
|
+
keyword_args["adapter_type"] = adapter_type # type: ignore
|
|
812
|
+
|
|
813
|
+
if checker(previous_node, node, **keyword_args): # type: ignore
|
|
814
|
+
yield unique_id
|
|
815
|
+
|
|
816
|
+
# checkers that can handle removed nodes
|
|
817
|
+
if checker.__name__ in [
|
|
818
|
+
"check_modified_contract",
|
|
819
|
+
"check_modified_content",
|
|
820
|
+
"check_unmodified_content",
|
|
821
|
+
]:
|
|
822
|
+
# ignore included_nodes, since those cannot contain removed nodes
|
|
823
|
+
for previous_unique_id, previous_node in manifest.nodes.items():
|
|
824
|
+
# detect removed (deleted, renamed, or disabled) nodes
|
|
825
|
+
removed_node = None
|
|
826
|
+
if previous_unique_id in self.manifest.disabled.keys():
|
|
827
|
+
removed_node = self.manifest.disabled[previous_unique_id][0]
|
|
828
|
+
elif previous_unique_id not in self.manifest.nodes.keys():
|
|
829
|
+
removed_node = previous_node
|
|
830
|
+
|
|
831
|
+
if removed_node:
|
|
832
|
+
# do not yield -- removed nodes should never be selected for downstream execution
|
|
833
|
+
# as they are not part of the current project's manifest.nodes
|
|
834
|
+
checker(removed_node, None, **keyword_args) # type: ignore
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
class ResultSelectorMethod(SelectorMethod):
|
|
838
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
839
|
+
if self.previous_state is None or self.previous_state.results is None:
|
|
840
|
+
raise DbtInternalError("No comparison run_results")
|
|
841
|
+
matches = set(
|
|
842
|
+
result.unique_id for result in self.previous_state.results if result.status == selector
|
|
843
|
+
)
|
|
844
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
845
|
+
if unique_id in matches:
|
|
846
|
+
yield unique_id
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
class SourceStatusSelectorMethod(SelectorMethod):
|
|
850
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
851
|
+
|
|
852
|
+
if self.previous_state is None or self.previous_state.sources is None:
|
|
853
|
+
raise DbtInternalError(
|
|
854
|
+
"No previous state comparison freshness results in sources.json"
|
|
855
|
+
)
|
|
856
|
+
elif self.previous_state.sources_current is None:
|
|
857
|
+
raise DbtInternalError("No current state comparison freshness results in sources.json")
|
|
858
|
+
|
|
859
|
+
current_state_sources = {
|
|
860
|
+
result.unique_id: getattr(result, "max_loaded_at", 0)
|
|
861
|
+
for result in self.previous_state.sources_current.results
|
|
862
|
+
if hasattr(result, "max_loaded_at")
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
current_state_sources_runtime_error = {
|
|
866
|
+
result.unique_id
|
|
867
|
+
for result in self.previous_state.sources_current.results
|
|
868
|
+
if not hasattr(result, "max_loaded_at")
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
previous_state_sources = {
|
|
872
|
+
result.unique_id: getattr(result, "max_loaded_at", 0)
|
|
873
|
+
for result in self.previous_state.sources.results
|
|
874
|
+
if hasattr(result, "max_loaded_at")
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
previous_state_sources_runtime_error = {
|
|
878
|
+
result.unique_id
|
|
879
|
+
for result in self.previous_state.sources_current.results
|
|
880
|
+
if not hasattr(result, "max_loaded_at")
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
matches = set()
|
|
884
|
+
if selector == "fresher":
|
|
885
|
+
for unique_id in current_state_sources:
|
|
886
|
+
if unique_id not in previous_state_sources:
|
|
887
|
+
matches.add(unique_id)
|
|
888
|
+
elif current_state_sources[unique_id] > previous_state_sources[unique_id]:
|
|
889
|
+
matches.add(unique_id)
|
|
890
|
+
|
|
891
|
+
for unique_id in matches:
|
|
892
|
+
if (
|
|
893
|
+
unique_id in previous_state_sources_runtime_error
|
|
894
|
+
or unique_id in current_state_sources_runtime_error
|
|
895
|
+
):
|
|
896
|
+
matches.remove(unique_id)
|
|
897
|
+
|
|
898
|
+
for unique_id, node in self.all_nodes(included_nodes):
|
|
899
|
+
if unique_id in matches:
|
|
900
|
+
yield unique_id
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
class VersionSelectorMethod(SelectorMethod):
|
|
904
|
+
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
|
905
|
+
for unique_id, node in self.parsed_nodes(included_nodes):
|
|
906
|
+
if isinstance(node, ModelNode):
|
|
907
|
+
if selector == "latest":
|
|
908
|
+
if node.is_latest_version:
|
|
909
|
+
yield unique_id
|
|
910
|
+
elif selector == "prerelease":
|
|
911
|
+
if (
|
|
912
|
+
node.version
|
|
913
|
+
and node.latest_version
|
|
914
|
+
and UnparsedVersion(v=node.version)
|
|
915
|
+
> UnparsedVersion(v=node.latest_version)
|
|
916
|
+
):
|
|
917
|
+
yield unique_id
|
|
918
|
+
elif selector == "old":
|
|
919
|
+
if (
|
|
920
|
+
node.version
|
|
921
|
+
and node.latest_version
|
|
922
|
+
and UnparsedVersion(v=node.version)
|
|
923
|
+
< UnparsedVersion(v=node.latest_version)
|
|
924
|
+
):
|
|
925
|
+
yield unique_id
|
|
926
|
+
elif selector == "none":
|
|
927
|
+
if node.version is None:
|
|
928
|
+
yield unique_id
|
|
929
|
+
else:
|
|
930
|
+
raise DbtRuntimeError(
|
|
931
|
+
f'Invalid version type selector {selector}: expected one of: "latest", "prerelease", "old", or "none"'
|
|
932
|
+
)
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
class MethodManager:
|
|
936
|
+
SELECTOR_METHODS: Dict[MethodName, Type[SelectorMethod]] = {
|
|
937
|
+
MethodName.FQN: QualifiedNameSelectorMethod,
|
|
938
|
+
MethodName.Tag: TagSelectorMethod,
|
|
939
|
+
MethodName.Group: GroupSelectorMethod,
|
|
940
|
+
MethodName.Access: AccessSelectorMethod,
|
|
941
|
+
MethodName.Source: SourceSelectorMethod,
|
|
942
|
+
MethodName.Path: PathSelectorMethod,
|
|
943
|
+
MethodName.File: FileSelectorMethod,
|
|
944
|
+
MethodName.Package: PackageSelectorMethod,
|
|
945
|
+
MethodName.Config: ConfigSelectorMethod,
|
|
946
|
+
MethodName.TestName: TestNameSelectorMethod,
|
|
947
|
+
MethodName.TestType: TestTypeSelectorMethod,
|
|
948
|
+
MethodName.ResourceType: ResourceTypeSelectorMethod,
|
|
949
|
+
MethodName.State: StateSelectorMethod,
|
|
950
|
+
MethodName.Exposure: ExposureSelectorMethod,
|
|
951
|
+
MethodName.Metric: MetricSelectorMethod,
|
|
952
|
+
MethodName.Result: ResultSelectorMethod,
|
|
953
|
+
MethodName.SourceStatus: SourceStatusSelectorMethod,
|
|
954
|
+
MethodName.Version: VersionSelectorMethod,
|
|
955
|
+
MethodName.SemanticModel: SemanticModelSelectorMethod,
|
|
956
|
+
MethodName.SavedQuery: SavedQuerySelectorMethod,
|
|
957
|
+
MethodName.UnitTest: UnitTestSelectorMethod,
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
def __init__(
|
|
961
|
+
self,
|
|
962
|
+
manifest: Manifest,
|
|
963
|
+
previous_state: Optional[PreviousState],
|
|
964
|
+
) -> None:
|
|
965
|
+
self.manifest = manifest
|
|
966
|
+
self.previous_state = previous_state
|
|
967
|
+
|
|
968
|
+
def get_method(self, method: MethodName, method_arguments: List[str]) -> SelectorMethod:
|
|
969
|
+
|
|
970
|
+
if method not in self.SELECTOR_METHODS:
|
|
971
|
+
raise DbtInternalError(
|
|
972
|
+
f'Method name "{method}" is a valid node selection '
|
|
973
|
+
f"method name, but it is not handled"
|
|
974
|
+
)
|
|
975
|
+
cls: Type[SelectorMethod] = self.SELECTOR_METHODS[method]
|
|
976
|
+
return cls(self.manifest, self.previous_state, method_arguments)
|