dvt-core 1.11.0b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dvt-core might be problematic. Click here for more details.
- dvt/__init__.py +7 -0
- dvt/_pydantic_shim.py +26 -0
- dvt/adapters/__init__.py +16 -0
- dvt/adapters/multi_adapter_manager.py +268 -0
- dvt/artifacts/__init__.py +0 -0
- dvt/artifacts/exceptions/__init__.py +1 -0
- dvt/artifacts/exceptions/schemas.py +31 -0
- dvt/artifacts/resources/__init__.py +116 -0
- dvt/artifacts/resources/base.py +68 -0
- dvt/artifacts/resources/types.py +93 -0
- dvt/artifacts/resources/v1/analysis.py +10 -0
- dvt/artifacts/resources/v1/catalog.py +23 -0
- dvt/artifacts/resources/v1/components.py +275 -0
- dvt/artifacts/resources/v1/config.py +282 -0
- dvt/artifacts/resources/v1/documentation.py +11 -0
- dvt/artifacts/resources/v1/exposure.py +52 -0
- dvt/artifacts/resources/v1/function.py +53 -0
- dvt/artifacts/resources/v1/generic_test.py +32 -0
- dvt/artifacts/resources/v1/group.py +22 -0
- dvt/artifacts/resources/v1/hook.py +11 -0
- dvt/artifacts/resources/v1/macro.py +30 -0
- dvt/artifacts/resources/v1/metric.py +173 -0
- dvt/artifacts/resources/v1/model.py +146 -0
- dvt/artifacts/resources/v1/owner.py +10 -0
- dvt/artifacts/resources/v1/saved_query.py +112 -0
- dvt/artifacts/resources/v1/seed.py +42 -0
- dvt/artifacts/resources/v1/semantic_layer_components.py +72 -0
- dvt/artifacts/resources/v1/semantic_model.py +315 -0
- dvt/artifacts/resources/v1/singular_test.py +14 -0
- dvt/artifacts/resources/v1/snapshot.py +92 -0
- dvt/artifacts/resources/v1/source_definition.py +85 -0
- dvt/artifacts/resources/v1/sql_operation.py +10 -0
- dvt/artifacts/resources/v1/unit_test_definition.py +78 -0
- dvt/artifacts/schemas/__init__.py +0 -0
- dvt/artifacts/schemas/base.py +191 -0
- dvt/artifacts/schemas/batch_results.py +24 -0
- dvt/artifacts/schemas/catalog/__init__.py +12 -0
- dvt/artifacts/schemas/catalog/v1/__init__.py +0 -0
- dvt/artifacts/schemas/catalog/v1/catalog.py +60 -0
- dvt/artifacts/schemas/freshness/__init__.py +1 -0
- dvt/artifacts/schemas/freshness/v3/__init__.py +0 -0
- dvt/artifacts/schemas/freshness/v3/freshness.py +159 -0
- dvt/artifacts/schemas/manifest/__init__.py +2 -0
- dvt/artifacts/schemas/manifest/v12/__init__.py +0 -0
- dvt/artifacts/schemas/manifest/v12/manifest.py +212 -0
- dvt/artifacts/schemas/results.py +148 -0
- dvt/artifacts/schemas/run/__init__.py +2 -0
- dvt/artifacts/schemas/run/v5/__init__.py +0 -0
- dvt/artifacts/schemas/run/v5/run.py +184 -0
- dvt/artifacts/schemas/upgrades/__init__.py +4 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
- dvt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
- dvt/artifacts/utils/validation.py +153 -0
- dvt/cli/__init__.py +1 -0
- dvt/cli/context.py +16 -0
- dvt/cli/exceptions.py +56 -0
- dvt/cli/flags.py +558 -0
- dvt/cli/main.py +971 -0
- dvt/cli/option_types.py +121 -0
- dvt/cli/options.py +79 -0
- dvt/cli/params.py +803 -0
- dvt/cli/requires.py +478 -0
- dvt/cli/resolvers.py +32 -0
- dvt/cli/types.py +40 -0
- dvt/clients/__init__.py +0 -0
- dvt/clients/checked_load.py +82 -0
- dvt/clients/git.py +164 -0
- dvt/clients/jinja.py +206 -0
- dvt/clients/jinja_static.py +245 -0
- dvt/clients/registry.py +192 -0
- dvt/clients/yaml_helper.py +68 -0
- dvt/compilation.py +833 -0
- dvt/compute/__init__.py +26 -0
- dvt/compute/base.py +288 -0
- dvt/compute/engines/__init__.py +13 -0
- dvt/compute/engines/duckdb_engine.py +368 -0
- dvt/compute/engines/spark_engine.py +273 -0
- dvt/compute/query_analyzer.py +212 -0
- dvt/compute/router.py +483 -0
- dvt/config/__init__.py +4 -0
- dvt/config/catalogs.py +95 -0
- dvt/config/compute_config.py +406 -0
- dvt/config/profile.py +411 -0
- dvt/config/profiles_v2.py +464 -0
- dvt/config/project.py +893 -0
- dvt/config/renderer.py +232 -0
- dvt/config/runtime.py +491 -0
- dvt/config/selectors.py +209 -0
- dvt/config/utils.py +78 -0
- dvt/connectors/.gitignore +6 -0
- dvt/connectors/README.md +306 -0
- dvt/connectors/catalog.yml +217 -0
- dvt/connectors/download_connectors.py +300 -0
- dvt/constants.py +29 -0
- dvt/context/__init__.py +0 -0
- dvt/context/base.py +746 -0
- dvt/context/configured.py +136 -0
- dvt/context/context_config.py +350 -0
- dvt/context/docs.py +82 -0
- dvt/context/exceptions_jinja.py +179 -0
- dvt/context/macro_resolver.py +195 -0
- dvt/context/macros.py +171 -0
- dvt/context/manifest.py +73 -0
- dvt/context/providers.py +2198 -0
- dvt/context/query_header.py +14 -0
- dvt/context/secret.py +59 -0
- dvt/context/target.py +74 -0
- dvt/contracts/__init__.py +0 -0
- dvt/contracts/files.py +413 -0
- dvt/contracts/graph/__init__.py +0 -0
- dvt/contracts/graph/manifest.py +1904 -0
- dvt/contracts/graph/metrics.py +98 -0
- dvt/contracts/graph/model_config.py +71 -0
- dvt/contracts/graph/node_args.py +42 -0
- dvt/contracts/graph/nodes.py +1806 -0
- dvt/contracts/graph/semantic_manifest.py +233 -0
- dvt/contracts/graph/unparsed.py +812 -0
- dvt/contracts/project.py +417 -0
- dvt/contracts/results.py +53 -0
- dvt/contracts/selection.py +23 -0
- dvt/contracts/sql.py +86 -0
- dvt/contracts/state.py +69 -0
- dvt/contracts/util.py +46 -0
- dvt/deprecations.py +347 -0
- dvt/deps/__init__.py +0 -0
- dvt/deps/base.py +153 -0
- dvt/deps/git.py +196 -0
- dvt/deps/local.py +80 -0
- dvt/deps/registry.py +131 -0
- dvt/deps/resolver.py +149 -0
- dvt/deps/tarball.py +121 -0
- dvt/docs/source/_ext/dbt_click.py +118 -0
- dvt/docs/source/conf.py +32 -0
- dvt/env_vars.py +64 -0
- dvt/event_time/event_time.py +40 -0
- dvt/event_time/sample_window.py +60 -0
- dvt/events/__init__.py +16 -0
- dvt/events/base_types.py +37 -0
- dvt/events/core_types_pb2.py +2 -0
- dvt/events/logging.py +109 -0
- dvt/events/types.py +2534 -0
- dvt/exceptions.py +1487 -0
- dvt/flags.py +89 -0
- dvt/graph/__init__.py +11 -0
- dvt/graph/cli.py +248 -0
- dvt/graph/graph.py +172 -0
- dvt/graph/queue.py +213 -0
- dvt/graph/selector.py +375 -0
- dvt/graph/selector_methods.py +976 -0
- dvt/graph/selector_spec.py +223 -0
- dvt/graph/thread_pool.py +18 -0
- dvt/hooks.py +21 -0
- dvt/include/README.md +49 -0
- dvt/include/__init__.py +3 -0
- dvt/include/global_project.py +4 -0
- dvt/include/starter_project/.gitignore +4 -0
- dvt/include/starter_project/README.md +15 -0
- dvt/include/starter_project/__init__.py +3 -0
- dvt/include/starter_project/analyses/.gitkeep +0 -0
- dvt/include/starter_project/dvt_project.yml +36 -0
- dvt/include/starter_project/macros/.gitkeep +0 -0
- dvt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
- dvt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
- dvt/include/starter_project/models/example/schema.yml +21 -0
- dvt/include/starter_project/seeds/.gitkeep +0 -0
- dvt/include/starter_project/snapshots/.gitkeep +0 -0
- dvt/include/starter_project/tests/.gitkeep +0 -0
- dvt/internal_deprecations.py +27 -0
- dvt/jsonschemas/__init__.py +3 -0
- dvt/jsonschemas/jsonschemas.py +309 -0
- dvt/jsonschemas/project/0.0.110.json +4717 -0
- dvt/jsonschemas/project/0.0.85.json +2015 -0
- dvt/jsonschemas/resources/0.0.110.json +2636 -0
- dvt/jsonschemas/resources/0.0.85.json +2536 -0
- dvt/jsonschemas/resources/latest.json +6773 -0
- dvt/links.py +4 -0
- dvt/materializations/__init__.py +0 -0
- dvt/materializations/incremental/__init__.py +0 -0
- dvt/materializations/incremental/microbatch.py +235 -0
- dvt/mp_context.py +8 -0
- dvt/node_types.py +37 -0
- dvt/parser/__init__.py +23 -0
- dvt/parser/analysis.py +21 -0
- dvt/parser/base.py +549 -0
- dvt/parser/common.py +267 -0
- dvt/parser/docs.py +52 -0
- dvt/parser/fixtures.py +51 -0
- dvt/parser/functions.py +30 -0
- dvt/parser/generic_test.py +100 -0
- dvt/parser/generic_test_builders.py +334 -0
- dvt/parser/hooks.py +119 -0
- dvt/parser/macros.py +137 -0
- dvt/parser/manifest.py +2204 -0
- dvt/parser/models.py +574 -0
- dvt/parser/partial.py +1179 -0
- dvt/parser/read_files.py +445 -0
- dvt/parser/schema_generic_tests.py +423 -0
- dvt/parser/schema_renderer.py +111 -0
- dvt/parser/schema_yaml_readers.py +936 -0
- dvt/parser/schemas.py +1467 -0
- dvt/parser/search.py +149 -0
- dvt/parser/seeds.py +28 -0
- dvt/parser/singular_test.py +20 -0
- dvt/parser/snapshots.py +44 -0
- dvt/parser/sources.py +557 -0
- dvt/parser/sql.py +63 -0
- dvt/parser/unit_tests.py +622 -0
- dvt/plugins/__init__.py +20 -0
- dvt/plugins/contracts.py +10 -0
- dvt/plugins/exceptions.py +2 -0
- dvt/plugins/manager.py +164 -0
- dvt/plugins/manifest.py +21 -0
- dvt/profiler.py +20 -0
- dvt/py.typed +1 -0
- dvt/runners/__init__.py +2 -0
- dvt/runners/exposure_runner.py +7 -0
- dvt/runners/no_op_runner.py +46 -0
- dvt/runners/saved_query_runner.py +7 -0
- dvt/selected_resources.py +8 -0
- dvt/task/__init__.py +0 -0
- dvt/task/base.py +504 -0
- dvt/task/build.py +197 -0
- dvt/task/clean.py +57 -0
- dvt/task/clone.py +162 -0
- dvt/task/compile.py +151 -0
- dvt/task/compute.py +366 -0
- dvt/task/debug.py +650 -0
- dvt/task/deps.py +280 -0
- dvt/task/docs/__init__.py +3 -0
- dvt/task/docs/generate.py +408 -0
- dvt/task/docs/index.html +250 -0
- dvt/task/docs/serve.py +28 -0
- dvt/task/freshness.py +323 -0
- dvt/task/function.py +122 -0
- dvt/task/group_lookup.py +46 -0
- dvt/task/init.py +374 -0
- dvt/task/list.py +237 -0
- dvt/task/printer.py +176 -0
- dvt/task/profiles.py +256 -0
- dvt/task/retry.py +175 -0
- dvt/task/run.py +1146 -0
- dvt/task/run_operation.py +142 -0
- dvt/task/runnable.py +802 -0
- dvt/task/seed.py +104 -0
- dvt/task/show.py +150 -0
- dvt/task/snapshot.py +57 -0
- dvt/task/sql.py +111 -0
- dvt/task/test.py +464 -0
- dvt/tests/fixtures/__init__.py +1 -0
- dvt/tests/fixtures/project.py +620 -0
- dvt/tests/util.py +651 -0
- dvt/tracking.py +529 -0
- dvt/utils/__init__.py +3 -0
- dvt/utils/artifact_upload.py +151 -0
- dvt/utils/utils.py +408 -0
- dvt/version.py +249 -0
- dvt_core-1.11.0b4.dist-info/METADATA +252 -0
- dvt_core-1.11.0b4.dist-info/RECORD +261 -0
- dvt_core-1.11.0b4.dist-info/WHEEL +5 -0
- dvt_core-1.11.0b4.dist-info/entry_points.txt +2 -0
- dvt_core-1.11.0b4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1806 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import os
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import (
|
|
7
|
+
Any,
|
|
8
|
+
Dict,
|
|
9
|
+
Iterator,
|
|
10
|
+
List,
|
|
11
|
+
Literal,
|
|
12
|
+
Optional,
|
|
13
|
+
Sequence,
|
|
14
|
+
Tuple,
|
|
15
|
+
Type,
|
|
16
|
+
Union,
|
|
17
|
+
get_args,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
from dvt.artifacts.resources import Analysis as AnalysisResource
|
|
21
|
+
from dvt.artifacts.resources import (
|
|
22
|
+
BaseResource,
|
|
23
|
+
ColumnInfo,
|
|
24
|
+
CompiledResource,
|
|
25
|
+
DependsOn,
|
|
26
|
+
Docs,
|
|
27
|
+
)
|
|
28
|
+
from dvt.artifacts.resources import Documentation as DocumentationResource
|
|
29
|
+
from dvt.artifacts.resources import Exposure as ExposureResource
|
|
30
|
+
from dvt.artifacts.resources import FileHash
|
|
31
|
+
from dvt.artifacts.resources import Function as FunctionResource
|
|
32
|
+
from dvt.artifacts.resources import FunctionArgument, FunctionReturns
|
|
33
|
+
from dvt.artifacts.resources import GenericTest as GenericTestResource
|
|
34
|
+
from dvt.artifacts.resources import GraphResource
|
|
35
|
+
from dvt.artifacts.resources import Group as GroupResource
|
|
36
|
+
from dvt.artifacts.resources import HasRelationMetadata as HasRelationMetadataResource
|
|
37
|
+
from dvt.artifacts.resources import HookNode as HookNodeResource
|
|
38
|
+
from dvt.artifacts.resources import InjectedCTE
|
|
39
|
+
from dvt.artifacts.resources import Macro as MacroResource
|
|
40
|
+
from dvt.artifacts.resources import MacroArgument
|
|
41
|
+
from dvt.artifacts.resources import Metric as MetricResource
|
|
42
|
+
from dvt.artifacts.resources import MetricInputMeasure
|
|
43
|
+
from dvt.artifacts.resources import Model as ModelResource
|
|
44
|
+
from dvt.artifacts.resources import (
|
|
45
|
+
ModelConfig,
|
|
46
|
+
ModelFreshness,
|
|
47
|
+
NodeConfig,
|
|
48
|
+
NodeVersion,
|
|
49
|
+
ParsedResource,
|
|
50
|
+
ParsedResourceMandatory,
|
|
51
|
+
)
|
|
52
|
+
from dvt.artifacts.resources import Quoting as QuotingResource
|
|
53
|
+
from dvt.artifacts.resources import SavedQuery as SavedQueryResource
|
|
54
|
+
from dvt.artifacts.resources import Seed as SeedResource
|
|
55
|
+
from dvt.artifacts.resources import SemanticModel as SemanticModelResource
|
|
56
|
+
from dvt.artifacts.resources import SingularTest as SingularTestResource
|
|
57
|
+
from dvt.artifacts.resources import Snapshot as SnapshotResource
|
|
58
|
+
from dvt.artifacts.resources import SourceDefinition as SourceDefinitionResource
|
|
59
|
+
from dvt.artifacts.resources import SqlOperation as SqlOperationResource
|
|
60
|
+
from dvt.artifacts.resources import TimeSpine
|
|
61
|
+
from dvt.artifacts.resources import UnitTestDefinition as UnitTestDefinitionResource
|
|
62
|
+
from dvt.artifacts.schemas.batch_results import BatchResults
|
|
63
|
+
from dvt.clients.jinja_static import statically_extract_has_name_this
|
|
64
|
+
from dvt.contracts.graph.model_config import UnitTestNodeConfig
|
|
65
|
+
from dvt.contracts.graph.node_args import ModelNodeArgs
|
|
66
|
+
from dvt.contracts.graph.unparsed import (
|
|
67
|
+
HasYamlMetadata,
|
|
68
|
+
TestDef,
|
|
69
|
+
UnitTestOverrides,
|
|
70
|
+
UnparsedColumn,
|
|
71
|
+
UnparsedSourceDefinition,
|
|
72
|
+
UnparsedSourceTableDefinition,
|
|
73
|
+
)
|
|
74
|
+
from dvt.events.types import (
|
|
75
|
+
SeedExceedsLimitAndPathChanged,
|
|
76
|
+
SeedExceedsLimitChecksumChanged,
|
|
77
|
+
SeedExceedsLimitSamePath,
|
|
78
|
+
SeedIncreased,
|
|
79
|
+
UnversionedBreakingChange,
|
|
80
|
+
)
|
|
81
|
+
from dvt.exceptions import ContractBreakingChangeError, ParsingError, ValidationError
|
|
82
|
+
from dvt.flags import get_flags
|
|
83
|
+
from dvt.node_types import (
|
|
84
|
+
REFABLE_NODE_TYPES,
|
|
85
|
+
VERSIONED_NODE_TYPES,
|
|
86
|
+
AccessType,
|
|
87
|
+
NodeType,
|
|
88
|
+
)
|
|
89
|
+
from mashumaro.types import SerializableType
|
|
90
|
+
|
|
91
|
+
from dbt.adapters.base import ConstraintSupport
|
|
92
|
+
from dbt.adapters.factory import get_adapter_constraint_support
|
|
93
|
+
from dbt_common.clients.system import write_file
|
|
94
|
+
from dbt_common.contracts.constraints import (
|
|
95
|
+
ColumnLevelConstraint,
|
|
96
|
+
ConstraintType,
|
|
97
|
+
ModelLevelConstraint,
|
|
98
|
+
)
|
|
99
|
+
from dbt_common.dataclass_schema import dbtClassMixin
|
|
100
|
+
from dbt_common.events.contextvars import set_log_contextvars
|
|
101
|
+
from dbt_common.events.functions import warn_or_error
|
|
102
|
+
|
|
103
|
+
# =====================================================================
|
|
104
|
+
# This contains the classes for all of the nodes and node-like objects
|
|
105
|
+
# in the manifest. In the "nodes" dictionary of the manifest we find
|
|
106
|
+
# all of the objects in the ManifestNode union below. In addition the
|
|
107
|
+
# manifest contains "macros", "sources", "metrics", "exposures", "docs",
|
|
108
|
+
# and "disabled" dictionaries.
|
|
109
|
+
#
|
|
110
|
+
# The SeedNode is a ManifestNode, but can't be compiled because it has
|
|
111
|
+
# no SQL.
|
|
112
|
+
#
|
|
113
|
+
# All objects defined in this file should have BaseNode as a parent
|
|
114
|
+
# class.
|
|
115
|
+
#
|
|
116
|
+
# The two objects which do not show up in the DAG are Macro and
|
|
117
|
+
# Documentation.
|
|
118
|
+
# =====================================================================
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ==================================================
|
|
122
|
+
# Various parent classes and node attribute classes
|
|
123
|
+
# ==================================================
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass
|
|
127
|
+
class BaseNode(BaseResource):
|
|
128
|
+
"""All nodes or node-like objects in this file should have this as a base class"""
|
|
129
|
+
|
|
130
|
+
# In an ideal world this would be a class property. However, chaining @classmethod and
|
|
131
|
+
# @property was deprecated in python 3.11 and removed in 3.13. There are more
|
|
132
|
+
# complicated ways of making a class property, however a class method suits our
|
|
133
|
+
# purposes well enough
|
|
134
|
+
@classmethod
|
|
135
|
+
def resource_class(cls) -> Type[BaseResource]:
|
|
136
|
+
"""Should be overriden by any class inheriting BaseNode"""
|
|
137
|
+
raise NotImplementedError
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def search_name(self):
|
|
141
|
+
return self.name
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def file_id(self):
|
|
145
|
+
return f"{self.package_name}://{self.original_file_path}"
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def is_refable(self):
|
|
149
|
+
return self.resource_type in REFABLE_NODE_TYPES
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def should_store_failures(self):
|
|
153
|
+
return False
|
|
154
|
+
|
|
155
|
+
# will this node map to an object in the database?
|
|
156
|
+
@property
|
|
157
|
+
def is_relational(self):
|
|
158
|
+
return self.resource_type in REFABLE_NODE_TYPES
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def is_versioned(self):
|
|
162
|
+
return self.resource_type in VERSIONED_NODE_TYPES and self.version is not None
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def is_ephemeral(self):
|
|
166
|
+
return self.config.materialized == "ephemeral"
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def is_ephemeral_model(self):
|
|
170
|
+
return self.is_refable and self.is_ephemeral
|
|
171
|
+
|
|
172
|
+
def get_materialization(self):
|
|
173
|
+
return self.config.materialized
|
|
174
|
+
|
|
175
|
+
@classmethod
|
|
176
|
+
def from_resource(cls, resource_instance: BaseResource):
|
|
177
|
+
assert isinstance(resource_instance, cls.resource_class())
|
|
178
|
+
return cls.from_dict(resource_instance.to_dict())
|
|
179
|
+
|
|
180
|
+
def to_resource(self):
|
|
181
|
+
return self.resource_class().from_dict(self.to_dict())
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass
|
|
185
|
+
class GraphNode(GraphResource, BaseNode):
|
|
186
|
+
"""Nodes in the DAG. Macro and Documentation don't have fqn."""
|
|
187
|
+
|
|
188
|
+
def same_fqn(self, other) -> bool:
|
|
189
|
+
return self.fqn == other.fqn
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@dataclass
|
|
193
|
+
class HasRelationMetadata(HasRelationMetadataResource):
|
|
194
|
+
@classmethod
|
|
195
|
+
def __pre_deserialize__(cls, data):
|
|
196
|
+
data = super().__pre_deserialize__(data)
|
|
197
|
+
if "database" not in data:
|
|
198
|
+
data["database"] = None
|
|
199
|
+
return data
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def quoting_dict(self) -> Dict[str, bool]:
|
|
203
|
+
if hasattr(self, "quoting"):
|
|
204
|
+
return self.quoting.to_dict(omit_none=True)
|
|
205
|
+
else:
|
|
206
|
+
return {}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@dataclass
|
|
210
|
+
class ParsedNodeMandatory(ParsedResourceMandatory, GraphNode, HasRelationMetadata):
|
|
211
|
+
pass
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# This needs to be in all ManifestNodes and also in SourceDefinition,
|
|
215
|
+
# because of "source freshness". Should not be in artifacts, because we
|
|
216
|
+
# don't write out _event_status.
|
|
217
|
+
@dataclass
|
|
218
|
+
class NodeInfoMixin:
|
|
219
|
+
_event_status: Dict[str, Any] = field(default_factory=dict)
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def node_info(self):
|
|
223
|
+
node_info = {
|
|
224
|
+
"node_path": getattr(self, "path", None),
|
|
225
|
+
"node_name": getattr(self, "name", None),
|
|
226
|
+
"unique_id": getattr(self, "unique_id", None),
|
|
227
|
+
"resource_type": str(getattr(self, "resource_type", "")),
|
|
228
|
+
"materialized": self.config.get("materialized"),
|
|
229
|
+
"node_status": str(self._event_status.get("node_status")),
|
|
230
|
+
"node_started_at": self._event_status.get("started_at"),
|
|
231
|
+
"node_finished_at": self._event_status.get("finished_at"),
|
|
232
|
+
"meta": getattr(self, "meta", {}),
|
|
233
|
+
"node_relation": {
|
|
234
|
+
"database": getattr(self, "database", None),
|
|
235
|
+
"schema": getattr(self, "schema", None),
|
|
236
|
+
"alias": getattr(self, "alias", None),
|
|
237
|
+
"relation_name": getattr(self, "relation_name", None),
|
|
238
|
+
},
|
|
239
|
+
"node_checksum": getattr(getattr(self, "checksum", None), "checksum", None),
|
|
240
|
+
}
|
|
241
|
+
return node_info
|
|
242
|
+
|
|
243
|
+
def update_event_status(self, **kwargs):
|
|
244
|
+
for k, v in kwargs.items():
|
|
245
|
+
self._event_status[k] = v
|
|
246
|
+
set_log_contextvars(node_info=self.node_info)
|
|
247
|
+
|
|
248
|
+
def clear_event_status(self):
|
|
249
|
+
self._event_status = dict()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@dataclass
|
|
253
|
+
class ParsedNode(ParsedResource, NodeInfoMixin, ParsedNodeMandatory, SerializableType):
|
|
254
|
+
def get_target_write_path(
|
|
255
|
+
self, target_path: str, subdirectory: str, split_suffix: Optional[str] = None
|
|
256
|
+
):
|
|
257
|
+
# This is called for both the "compiled" subdirectory of "target" and the "run" subdirectory
|
|
258
|
+
if os.path.basename(self.path) == os.path.basename(self.original_file_path):
|
|
259
|
+
# One-to-one relationship of nodes to files.
|
|
260
|
+
path = self.original_file_path
|
|
261
|
+
else:
|
|
262
|
+
# Many-to-one relationship of nodes to files.
|
|
263
|
+
path = os.path.join(self.original_file_path, self.path)
|
|
264
|
+
|
|
265
|
+
if split_suffix:
|
|
266
|
+
pathlib_path = Path(path)
|
|
267
|
+
path = str(
|
|
268
|
+
pathlib_path.parent
|
|
269
|
+
/ pathlib_path.stem
|
|
270
|
+
/ (pathlib_path.stem + f"_{split_suffix}" + pathlib_path.suffix)
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
target_write_path = os.path.join(target_path, subdirectory, self.package_name, path)
|
|
274
|
+
return target_write_path
|
|
275
|
+
|
|
276
|
+
def write_node(self, project_root: str, compiled_path, compiled_code: str):
|
|
277
|
+
if os.path.isabs(compiled_path):
|
|
278
|
+
full_path = compiled_path
|
|
279
|
+
else:
|
|
280
|
+
full_path = os.path.join(project_root, compiled_path)
|
|
281
|
+
write_file(full_path, compiled_code)
|
|
282
|
+
|
|
283
|
+
def _serialize(self):
|
|
284
|
+
return self.to_dict()
|
|
285
|
+
|
|
286
|
+
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
|
287
|
+
dct = super().__post_serialize__(dct, context)
|
|
288
|
+
if "_event_status" in dct:
|
|
289
|
+
del dct["_event_status"]
|
|
290
|
+
return dct
|
|
291
|
+
|
|
292
|
+
@classmethod
|
|
293
|
+
def _deserialize(cls, dct: Dict[str, int]):
|
|
294
|
+
# The serialized ParsedNodes do not differ from each other
|
|
295
|
+
# in fields that would allow 'from_dict' to distinguis
|
|
296
|
+
# between them.
|
|
297
|
+
resource_type = dct["resource_type"]
|
|
298
|
+
if resource_type == "model":
|
|
299
|
+
return ModelNode.from_dict(dct)
|
|
300
|
+
elif resource_type == "analysis":
|
|
301
|
+
return AnalysisNode.from_dict(dct)
|
|
302
|
+
elif resource_type == "seed":
|
|
303
|
+
return SeedNode.from_dict(dct)
|
|
304
|
+
elif resource_type == "sql":
|
|
305
|
+
return SqlNode.from_dict(dct)
|
|
306
|
+
elif resource_type == "test":
|
|
307
|
+
if "test_metadata" in dct:
|
|
308
|
+
return GenericTestNode.from_dict(dct)
|
|
309
|
+
else:
|
|
310
|
+
return SingularTestNode.from_dict(dct)
|
|
311
|
+
elif resource_type == "operation":
|
|
312
|
+
return HookNode.from_dict(dct)
|
|
313
|
+
elif resource_type == "seed":
|
|
314
|
+
return SeedNode.from_dict(dct)
|
|
315
|
+
elif resource_type == "snapshot":
|
|
316
|
+
return SnapshotNode.from_dict(dct)
|
|
317
|
+
else:
|
|
318
|
+
return cls.from_dict(dct)
|
|
319
|
+
|
|
320
|
+
def _persist_column_docs(self) -> bool:
|
|
321
|
+
if hasattr(self.config, "persist_docs"):
|
|
322
|
+
assert isinstance(self.config, NodeConfig)
|
|
323
|
+
return bool(self.config.persist_docs.get("columns"))
|
|
324
|
+
return False
|
|
325
|
+
|
|
326
|
+
def _persist_relation_docs(self) -> bool:
|
|
327
|
+
if hasattr(self.config, "persist_docs"):
|
|
328
|
+
assert isinstance(self.config, NodeConfig)
|
|
329
|
+
return bool(self.config.persist_docs.get("relation"))
|
|
330
|
+
return False
|
|
331
|
+
|
|
332
|
+
def same_persisted_description(self, other) -> bool:
|
|
333
|
+
# the check on configs will handle the case where we have different
|
|
334
|
+
# persist settings, so we only have to care about the cases where they
|
|
335
|
+
# are the same..
|
|
336
|
+
if self._persist_relation_docs():
|
|
337
|
+
if self.description != other.description:
|
|
338
|
+
return False
|
|
339
|
+
|
|
340
|
+
if self._persist_column_docs():
|
|
341
|
+
# assert other._persist_column_docs()
|
|
342
|
+
column_descriptions = {k: v.description for k, v in self.columns.items()}
|
|
343
|
+
other_column_descriptions = {k: v.description for k, v in other.columns.items()}
|
|
344
|
+
if column_descriptions != other_column_descriptions:
|
|
345
|
+
return False
|
|
346
|
+
|
|
347
|
+
return True
|
|
348
|
+
|
|
349
|
+
def same_body(self, other) -> bool:
|
|
350
|
+
return self.raw_code == other.raw_code
|
|
351
|
+
|
|
352
|
+
def same_database_representation(self, other) -> bool:
|
|
353
|
+
# compare the config representation, not the node's config value. This
|
|
354
|
+
# compares the configured value, rather than the ultimate value (so
|
|
355
|
+
# generate_*_name and unset values derived from the target are
|
|
356
|
+
# ignored)
|
|
357
|
+
keys = ("database", "schema", "alias")
|
|
358
|
+
for key in keys:
|
|
359
|
+
mine = self.unrendered_config.get(key)
|
|
360
|
+
others = other.unrendered_config.get(key)
|
|
361
|
+
if mine != others:
|
|
362
|
+
return False
|
|
363
|
+
return True
|
|
364
|
+
|
|
365
|
+
def same_config(self, old) -> bool:
|
|
366
|
+
return self.config.same_contents(
|
|
367
|
+
self.unrendered_config,
|
|
368
|
+
old.unrendered_config,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def build_contract_checksum(self):
|
|
372
|
+
pass
|
|
373
|
+
|
|
374
|
+
def same_contract(self, old, adapter_type=None) -> bool:
|
|
375
|
+
# This would only apply to seeds
|
|
376
|
+
return True
|
|
377
|
+
|
|
378
|
+
def same_contents(self, old, adapter_type) -> bool:
|
|
379
|
+
if old is None:
|
|
380
|
+
return False
|
|
381
|
+
|
|
382
|
+
# Need to ensure that same_contract is called because it
|
|
383
|
+
# could throw an error
|
|
384
|
+
same_contract = self.same_contract(old, adapter_type)
|
|
385
|
+
return (
|
|
386
|
+
self.same_body(old)
|
|
387
|
+
and self.same_config(old)
|
|
388
|
+
and self.same_persisted_description(old)
|
|
389
|
+
and self.same_fqn(old)
|
|
390
|
+
and self.same_database_representation(old)
|
|
391
|
+
and same_contract
|
|
392
|
+
and True
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
@property
|
|
396
|
+
def is_external_node(self):
|
|
397
|
+
return False
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
@dataclass
|
|
401
|
+
class CompiledNode(CompiledResource, ParsedNode):
|
|
402
|
+
"""Contains attributes necessary for SQL files and nodes with refs, sources, etc,
|
|
403
|
+
so all ManifestNodes except SeedNode."""
|
|
404
|
+
|
|
405
|
+
@property
|
|
406
|
+
def empty(self):
|
|
407
|
+
return not self.raw_code.strip()
|
|
408
|
+
|
|
409
|
+
def set_cte(self, cte_id: str, sql: str):
|
|
410
|
+
"""This is the equivalent of what self.extra_ctes[cte_id] = sql would
|
|
411
|
+
do if extra_ctes were an OrderedDict
|
|
412
|
+
"""
|
|
413
|
+
for cte in self.extra_ctes:
|
|
414
|
+
# Because it's possible that multiple threads are compiling the
|
|
415
|
+
# node at the same time, we don't want to overwrite already compiled
|
|
416
|
+
# sql in the extra_ctes with empty sql.
|
|
417
|
+
if cte.id == cte_id:
|
|
418
|
+
break
|
|
419
|
+
else:
|
|
420
|
+
self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
|
|
421
|
+
|
|
422
|
+
@property
|
|
423
|
+
def depends_on_nodes(self):
|
|
424
|
+
return self.depends_on.nodes
|
|
425
|
+
|
|
426
|
+
@property
|
|
427
|
+
def depends_on_macros(self):
|
|
428
|
+
return self.depends_on.macros
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# ====================================
|
|
432
|
+
# CompiledNode subclasses
|
|
433
|
+
# ====================================
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
@dataclass
|
|
437
|
+
class AnalysisNode(AnalysisResource, CompiledNode):
|
|
438
|
+
@classmethod
|
|
439
|
+
def resource_class(cls) -> Type[AnalysisResource]:
|
|
440
|
+
return AnalysisResource
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
@dataclass
|
|
444
|
+
class HookNode(HookNodeResource, CompiledNode):
|
|
445
|
+
@classmethod
|
|
446
|
+
def resource_class(cls) -> Type[HookNodeResource]:
|
|
447
|
+
return HookNodeResource
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@dataclass
|
|
451
|
+
class BatchContext(dbtClassMixin):
|
|
452
|
+
id: str
|
|
453
|
+
event_time_start: datetime
|
|
454
|
+
event_time_end: datetime
|
|
455
|
+
|
|
456
|
+
def __post_serialize__(self, data, context):
|
|
457
|
+
# This is insane, but necessary, I apologize. Mashumaro handles the
|
|
458
|
+
# dictification of this class via a compile time generated `to_dict`
|
|
459
|
+
# method based off of the _typing_ of th class. By default `datetime`
|
|
460
|
+
# types are converted to strings. We don't want that, we want them to
|
|
461
|
+
# stay datetimes.
|
|
462
|
+
# Note: This is safe because the `BatchContext` isn't part of the artifact
|
|
463
|
+
# and thus doesn't get written out.
|
|
464
|
+
new_data = super().__post_serialize__(data, context)
|
|
465
|
+
new_data["event_time_start"] = self.event_time_start
|
|
466
|
+
new_data["event_time_end"] = self.event_time_end
|
|
467
|
+
return new_data
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
@dataclass
|
|
471
|
+
class ModelNode(ModelResource, CompiledNode):
|
|
472
|
+
previous_batch_results: Optional[BatchResults] = None
|
|
473
|
+
batch: Optional[BatchContext] = None
|
|
474
|
+
_has_this: Optional[bool] = None
|
|
475
|
+
|
|
476
|
+
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
|
477
|
+
dct = super().__post_serialize__(dct, context)
|
|
478
|
+
if "_has_this" in dct:
|
|
479
|
+
del dct["_has_this"]
|
|
480
|
+
if "previous_batch_results" in dct:
|
|
481
|
+
del dct["previous_batch_results"]
|
|
482
|
+
return dct
|
|
483
|
+
|
|
484
|
+
@classmethod
|
|
485
|
+
def resource_class(cls) -> Type[ModelResource]:
|
|
486
|
+
return ModelResource
|
|
487
|
+
|
|
488
|
+
@classmethod
|
|
489
|
+
def from_args(cls, args: ModelNodeArgs) -> "ModelNode":
|
|
490
|
+
unique_id = args.unique_id
|
|
491
|
+
|
|
492
|
+
# build unrendered config -- for usage in ParsedNode.same_contents
|
|
493
|
+
unrendered_config = {}
|
|
494
|
+
unrendered_config["alias"] = args.identifier
|
|
495
|
+
unrendered_config["schema"] = args.schema
|
|
496
|
+
if args.database:
|
|
497
|
+
unrendered_config["database"] = args.database
|
|
498
|
+
|
|
499
|
+
return cls(
|
|
500
|
+
resource_type=NodeType.Model,
|
|
501
|
+
name=args.name,
|
|
502
|
+
package_name=args.package_name,
|
|
503
|
+
unique_id=unique_id,
|
|
504
|
+
fqn=args.fqn,
|
|
505
|
+
version=args.version,
|
|
506
|
+
latest_version=args.latest_version,
|
|
507
|
+
relation_name=args.relation_name,
|
|
508
|
+
database=args.database,
|
|
509
|
+
schema=args.schema,
|
|
510
|
+
alias=args.identifier,
|
|
511
|
+
deprecation_date=args.deprecation_date,
|
|
512
|
+
checksum=FileHash.from_contents(f"{unique_id},{args.generated_at}"),
|
|
513
|
+
access=AccessType(args.access),
|
|
514
|
+
original_file_path="",
|
|
515
|
+
path="",
|
|
516
|
+
unrendered_config=unrendered_config,
|
|
517
|
+
depends_on=DependsOn(nodes=args.depends_on_nodes),
|
|
518
|
+
config=ModelConfig(enabled=args.enabled),
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
@property
|
|
522
|
+
def is_external_node(self) -> bool:
|
|
523
|
+
return not self.original_file_path and not self.path
|
|
524
|
+
|
|
525
|
+
@property
|
|
526
|
+
def is_latest_version(self) -> bool:
|
|
527
|
+
return self.version is not None and self.version == self.latest_version
|
|
528
|
+
|
|
529
|
+
@property
|
|
530
|
+
def is_past_deprecation_date(self) -> bool:
|
|
531
|
+
return (
|
|
532
|
+
self.deprecation_date is not None
|
|
533
|
+
and self.deprecation_date < datetime.now().astimezone()
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
@property
|
|
537
|
+
def search_name(self):
|
|
538
|
+
if self.version is None:
|
|
539
|
+
return self.name
|
|
540
|
+
else:
|
|
541
|
+
return f"{self.name}.v{self.version}"
|
|
542
|
+
|
|
543
|
+
@property
|
|
544
|
+
def materialization_enforces_constraints(self) -> bool:
|
|
545
|
+
return self.config.materialized in ["table", "incremental"]
|
|
546
|
+
|
|
547
|
+
@property
|
|
548
|
+
def all_constraints(self) -> List[Union[ModelLevelConstraint, ColumnLevelConstraint]]:
|
|
549
|
+
constraints: List[Union[ModelLevelConstraint, ColumnLevelConstraint]] = []
|
|
550
|
+
for model_level_constraint in self.constraints:
|
|
551
|
+
constraints.append(model_level_constraint)
|
|
552
|
+
|
|
553
|
+
for column in self.columns.values():
|
|
554
|
+
for column_level_constraint in column.constraints:
|
|
555
|
+
constraints.append(column_level_constraint)
|
|
556
|
+
|
|
557
|
+
return constraints
|
|
558
|
+
|
|
559
|
+
@property
|
|
560
|
+
def has_this(self) -> bool:
|
|
561
|
+
if self._has_this is None:
|
|
562
|
+
self._has_this = statically_extract_has_name_this(self.raw_code)
|
|
563
|
+
return self._has_this
|
|
564
|
+
|
|
565
|
+
def infer_primary_key(self, data_tests: List["GenericTestNode"]) -> List[str]:
|
|
566
|
+
"""
|
|
567
|
+
Infers the columns that can be used as primary key of a model in the following order:
|
|
568
|
+
1. Columns with primary key constraints
|
|
569
|
+
2. Columns with unique and not_null data tests
|
|
570
|
+
3. Columns with enabled unique or dbt_utils.unique_combination_of_columns data tests
|
|
571
|
+
4. Columns with disabled unique or dbt_utils.unique_combination_of_columns data tests
|
|
572
|
+
"""
|
|
573
|
+
for constraint in self.constraints:
|
|
574
|
+
if constraint.type == ConstraintType.primary_key:
|
|
575
|
+
return constraint.columns
|
|
576
|
+
|
|
577
|
+
for column, column_info in self.columns.items():
|
|
578
|
+
for column_constraint in column_info.constraints:
|
|
579
|
+
if column_constraint.type == ConstraintType.primary_key:
|
|
580
|
+
return [column]
|
|
581
|
+
|
|
582
|
+
columns_with_enabled_unique_tests = set()
|
|
583
|
+
columns_with_disabled_unique_tests = set()
|
|
584
|
+
columns_with_not_null_tests = set()
|
|
585
|
+
for test in data_tests:
|
|
586
|
+
columns: List[str] = []
|
|
587
|
+
# extract columns from test kwargs, ensuring columns is a List[str] given tests can have custom (user or pacakge-defined) kwarg types
|
|
588
|
+
if "column_name" in test.test_metadata.kwargs and isinstance(
|
|
589
|
+
test.test_metadata.kwargs["column_name"], str
|
|
590
|
+
):
|
|
591
|
+
columns = [test.test_metadata.kwargs["column_name"]]
|
|
592
|
+
elif "combination_of_columns" in test.test_metadata.kwargs and isinstance(
|
|
593
|
+
test.test_metadata.kwargs["combination_of_columns"], list
|
|
594
|
+
):
|
|
595
|
+
columns = [
|
|
596
|
+
column
|
|
597
|
+
for column in test.test_metadata.kwargs["combination_of_columns"]
|
|
598
|
+
if isinstance(column, str)
|
|
599
|
+
]
|
|
600
|
+
|
|
601
|
+
for column in columns:
|
|
602
|
+
if test.test_metadata.name in ["unique", "unique_combination_of_columns"]:
|
|
603
|
+
if test.config.enabled:
|
|
604
|
+
columns_with_enabled_unique_tests.add(column)
|
|
605
|
+
else:
|
|
606
|
+
columns_with_disabled_unique_tests.add(column)
|
|
607
|
+
elif test.test_metadata.name == "not_null":
|
|
608
|
+
columns_with_not_null_tests.add(column)
|
|
609
|
+
|
|
610
|
+
columns_with_unique_and_not_null_tests = []
|
|
611
|
+
for column in columns_with_not_null_tests:
|
|
612
|
+
if (
|
|
613
|
+
column in columns_with_enabled_unique_tests
|
|
614
|
+
or column in columns_with_disabled_unique_tests
|
|
615
|
+
):
|
|
616
|
+
columns_with_unique_and_not_null_tests.append(column)
|
|
617
|
+
if columns_with_unique_and_not_null_tests:
|
|
618
|
+
return columns_with_unique_and_not_null_tests
|
|
619
|
+
|
|
620
|
+
if columns_with_enabled_unique_tests:
|
|
621
|
+
return list(columns_with_enabled_unique_tests)
|
|
622
|
+
|
|
623
|
+
if columns_with_disabled_unique_tests:
|
|
624
|
+
return list(columns_with_disabled_unique_tests)
|
|
625
|
+
|
|
626
|
+
return []
|
|
627
|
+
|
|
628
|
+
def same_contents(self, old, adapter_type) -> bool:
|
|
629
|
+
return super().same_contents(old, adapter_type) and self.same_ref_representation(old)
|
|
630
|
+
|
|
631
|
+
def same_ref_representation(self, old) -> bool:
|
|
632
|
+
return (
|
|
633
|
+
# Changing the latest_version may break downstream unpinned refs
|
|
634
|
+
self.latest_version == old.latest_version
|
|
635
|
+
# Changes to access or deprecation_date may lead to ref-related parsing errors
|
|
636
|
+
and self.access == old.access
|
|
637
|
+
and self.deprecation_date == old.deprecation_date
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
def build_contract_checksum(self):
|
|
641
|
+
# We don't need to construct the checksum if the model does not
|
|
642
|
+
# have contract enforced, because it won't be used.
|
|
643
|
+
# This needs to be executed after contract config is set
|
|
644
|
+
|
|
645
|
+
# Avoid rebuilding the checksum if it has already been set.
|
|
646
|
+
if self.contract.checksum is not None:
|
|
647
|
+
return
|
|
648
|
+
|
|
649
|
+
if self.contract.enforced is True:
|
|
650
|
+
contract_state = ""
|
|
651
|
+
# We need to sort the columns so that order doesn't matter
|
|
652
|
+
# columns is a str: ColumnInfo dictionary
|
|
653
|
+
sorted_columns = sorted(self.columns.values(), key=lambda col: col.name)
|
|
654
|
+
for column in sorted_columns:
|
|
655
|
+
contract_state += f"|{column.name}"
|
|
656
|
+
contract_state += str(column.data_type)
|
|
657
|
+
contract_state += str(column.constraints)
|
|
658
|
+
if self.materialization_enforces_constraints:
|
|
659
|
+
contract_state += self.config.materialized
|
|
660
|
+
contract_state += str(self.constraints)
|
|
661
|
+
data = contract_state.encode("utf-8")
|
|
662
|
+
self.contract.checksum = hashlib.new("sha256", data).hexdigest()
|
|
663
|
+
|
|
664
|
+
def same_contract_removed(self) -> bool:
|
|
665
|
+
"""
|
|
666
|
+
self: the removed (deleted, renamed, or disabled) model node
|
|
667
|
+
"""
|
|
668
|
+
# If the contract wasn't previously enforced, no contract change has occurred
|
|
669
|
+
if self.contract.enforced is False:
|
|
670
|
+
return True
|
|
671
|
+
|
|
672
|
+
# Removed node is past its deprecation_date, so deletion does not constitute a contract change
|
|
673
|
+
if self.is_past_deprecation_date:
|
|
674
|
+
return True
|
|
675
|
+
|
|
676
|
+
# Disabled, deleted, or renamed node with previously enforced contract.
|
|
677
|
+
if not self.config.enabled:
|
|
678
|
+
breaking_change = f"Contracted model '{self.unique_id}' was disabled."
|
|
679
|
+
else:
|
|
680
|
+
breaking_change = f"Contracted model '{self.unique_id}' was deleted or renamed."
|
|
681
|
+
|
|
682
|
+
if self.version is None:
|
|
683
|
+
warn_or_error(
|
|
684
|
+
UnversionedBreakingChange(
|
|
685
|
+
breaking_changes=[breaking_change],
|
|
686
|
+
model_name=self.name,
|
|
687
|
+
model_file_path=self.original_file_path,
|
|
688
|
+
),
|
|
689
|
+
node=self,
|
|
690
|
+
)
|
|
691
|
+
return False
|
|
692
|
+
else:
|
|
693
|
+
raise (
|
|
694
|
+
ContractBreakingChangeError(
|
|
695
|
+
breaking_changes=[breaking_change],
|
|
696
|
+
node=self,
|
|
697
|
+
)
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
def same_contract(self, old, adapter_type=None) -> bool:
|
|
701
|
+
# If the contract wasn't previously enforced:
|
|
702
|
+
if old.contract.enforced is False and self.contract.enforced is False:
|
|
703
|
+
# No change -- same_contract: True
|
|
704
|
+
return True
|
|
705
|
+
if old.contract.enforced is False and self.contract.enforced is True:
|
|
706
|
+
# Now it's enforced. This is a change, but not a breaking change -- same_contract: False
|
|
707
|
+
return False
|
|
708
|
+
|
|
709
|
+
# Otherwise: The contract was previously enforced, and we need to check for changes.
|
|
710
|
+
# Happy path: The contract is still being enforced, and the checksums are identical.
|
|
711
|
+
if self.contract.enforced is True and self.contract.checksum == old.contract.checksum:
|
|
712
|
+
# No change -- same_contract: True
|
|
713
|
+
return True
|
|
714
|
+
|
|
715
|
+
# Otherwise: There has been a change.
|
|
716
|
+
# We need to determine if it is a **breaking** change.
|
|
717
|
+
# These are the categories of breaking changes:
|
|
718
|
+
contract_enforced_disabled: bool = False
|
|
719
|
+
columns_removed: List[str] = []
|
|
720
|
+
column_type_changes: List[Dict[str, str]] = []
|
|
721
|
+
enforced_column_constraint_removed: List[Dict[str, str]] = (
|
|
722
|
+
[]
|
|
723
|
+
) # column_name, constraint_type
|
|
724
|
+
enforced_model_constraint_removed: List[Dict[str, Any]] = [] # constraint_type, columns
|
|
725
|
+
materialization_changed: List[str] = []
|
|
726
|
+
|
|
727
|
+
if old.contract.enforced is True and self.contract.enforced is False:
|
|
728
|
+
# Breaking change: the contract was previously enforced, and it no longer is
|
|
729
|
+
contract_enforced_disabled = True
|
|
730
|
+
|
|
731
|
+
constraint_support = get_adapter_constraint_support(adapter_type)
|
|
732
|
+
column_constraints_exist = False
|
|
733
|
+
|
|
734
|
+
# Next, compare each column from the previous contract (old.columns)
|
|
735
|
+
for old_key, old_value in sorted(old.columns.items()):
|
|
736
|
+
# Has this column been removed?
|
|
737
|
+
if old_key not in self.columns.keys():
|
|
738
|
+
columns_removed.append(old_value.name)
|
|
739
|
+
# Has this column's data type changed?
|
|
740
|
+
elif old_value.data_type != self.columns[old_key].data_type:
|
|
741
|
+
column_type_changes.append(
|
|
742
|
+
{
|
|
743
|
+
"column_name": str(old_value.name),
|
|
744
|
+
"previous_column_type": str(old_value.data_type),
|
|
745
|
+
"current_column_type": str(self.columns[old_key].data_type),
|
|
746
|
+
}
|
|
747
|
+
)
|
|
748
|
+
|
|
749
|
+
# track if there are any column level constraints for the materialization check late
|
|
750
|
+
if old_value.constraints:
|
|
751
|
+
column_constraints_exist = True
|
|
752
|
+
|
|
753
|
+
# Have enforced columns level constraints changed?
|
|
754
|
+
# Constraints are only enforced for table and incremental materializations.
|
|
755
|
+
# We only really care if the old node was one of those materializations for breaking changes
|
|
756
|
+
if (
|
|
757
|
+
old_key in self.columns.keys()
|
|
758
|
+
and old_value.constraints != self.columns[old_key].constraints
|
|
759
|
+
and old.materialization_enforces_constraints
|
|
760
|
+
):
|
|
761
|
+
for old_constraint in old_value.constraints:
|
|
762
|
+
if (
|
|
763
|
+
old_constraint not in self.columns[old_key].constraints
|
|
764
|
+
and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED
|
|
765
|
+
):
|
|
766
|
+
enforced_column_constraint_removed.append(
|
|
767
|
+
{
|
|
768
|
+
"column_name": old_key,
|
|
769
|
+
"constraint_name": old_constraint.name,
|
|
770
|
+
"constraint_type": ConstraintType(old_constraint.type),
|
|
771
|
+
}
|
|
772
|
+
)
|
|
773
|
+
|
|
774
|
+
# Now compare the model level constraints
|
|
775
|
+
if old.constraints != self.constraints and old.materialization_enforces_constraints:
|
|
776
|
+
for old_constraint in old.constraints:
|
|
777
|
+
if (
|
|
778
|
+
old_constraint not in self.constraints
|
|
779
|
+
and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED
|
|
780
|
+
):
|
|
781
|
+
enforced_model_constraint_removed.append(
|
|
782
|
+
{
|
|
783
|
+
"constraint_name": old_constraint.name,
|
|
784
|
+
"constraint_type": ConstraintType(old_constraint.type),
|
|
785
|
+
"columns": old_constraint.columns,
|
|
786
|
+
}
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
# Check for relevant materialization changes.
|
|
790
|
+
if (
|
|
791
|
+
old.materialization_enforces_constraints
|
|
792
|
+
and not self.materialization_enforces_constraints
|
|
793
|
+
and (old.constraints or column_constraints_exist)
|
|
794
|
+
):
|
|
795
|
+
materialization_changed = [old.config.materialized, self.config.materialized]
|
|
796
|
+
|
|
797
|
+
# If a column has been added, it will be missing in the old.columns, and present in self.columns
|
|
798
|
+
# That's a change (caught by the different checksums), but not a breaking change
|
|
799
|
+
|
|
800
|
+
# Did we find any changes that we consider breaking? If there's an enforced contract, that's
|
|
801
|
+
# a warning unless the model is versioned, then it's an error.
|
|
802
|
+
if (
|
|
803
|
+
contract_enforced_disabled
|
|
804
|
+
or columns_removed
|
|
805
|
+
or column_type_changes
|
|
806
|
+
or enforced_model_constraint_removed
|
|
807
|
+
or enforced_column_constraint_removed
|
|
808
|
+
or materialization_changed
|
|
809
|
+
):
|
|
810
|
+
|
|
811
|
+
breaking_changes = []
|
|
812
|
+
if contract_enforced_disabled:
|
|
813
|
+
breaking_changes.append(
|
|
814
|
+
"Contract enforcement was removed: Previously, this model had an enforced contract. It is no longer configured to enforce its contract, and this is a breaking change."
|
|
815
|
+
)
|
|
816
|
+
if columns_removed:
|
|
817
|
+
columns_removed_str = "\n - ".join(columns_removed)
|
|
818
|
+
breaking_changes.append(f"Columns were removed: \n - {columns_removed_str}")
|
|
819
|
+
if column_type_changes:
|
|
820
|
+
column_type_changes_str = "\n - ".join(
|
|
821
|
+
[
|
|
822
|
+
f"{c['column_name']} ({c['previous_column_type']} -> {c['current_column_type']})"
|
|
823
|
+
for c in column_type_changes
|
|
824
|
+
]
|
|
825
|
+
)
|
|
826
|
+
breaking_changes.append(
|
|
827
|
+
f"Columns with data_type changes: \n - {column_type_changes_str}"
|
|
828
|
+
)
|
|
829
|
+
if enforced_column_constraint_removed:
|
|
830
|
+
column_constraint_changes_str = "\n - ".join(
|
|
831
|
+
[
|
|
832
|
+
f"'{c['constraint_name'] if c['constraint_name'] is not None else c['constraint_type']}' constraint on column {c['column_name']}"
|
|
833
|
+
for c in enforced_column_constraint_removed
|
|
834
|
+
]
|
|
835
|
+
)
|
|
836
|
+
breaking_changes.append(
|
|
837
|
+
f"Enforced column level constraints were removed: \n - {column_constraint_changes_str}"
|
|
838
|
+
)
|
|
839
|
+
if enforced_model_constraint_removed:
|
|
840
|
+
model_constraint_changes_str = "\n - ".join(
|
|
841
|
+
[
|
|
842
|
+
f"'{c['constraint_name'] if c['constraint_name'] is not None else c['constraint_type']}' constraint on columns {c['columns']}"
|
|
843
|
+
for c in enforced_model_constraint_removed
|
|
844
|
+
]
|
|
845
|
+
)
|
|
846
|
+
breaking_changes.append(
|
|
847
|
+
f"Enforced model level constraints were removed: \n - {model_constraint_changes_str}"
|
|
848
|
+
)
|
|
849
|
+
if materialization_changed:
|
|
850
|
+
materialization_changes_str = (
|
|
851
|
+
f"{materialization_changed[0]} -> {materialization_changed[1]}"
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
breaking_changes.append(
|
|
855
|
+
f"Materialization changed with enforced constraints: \n - {materialization_changes_str}"
|
|
856
|
+
)
|
|
857
|
+
|
|
858
|
+
if self.version is None:
|
|
859
|
+
warn_or_error(
|
|
860
|
+
UnversionedBreakingChange(
|
|
861
|
+
contract_enforced_disabled=contract_enforced_disabled,
|
|
862
|
+
columns_removed=columns_removed,
|
|
863
|
+
column_type_changes=column_type_changes,
|
|
864
|
+
enforced_column_constraint_removed=enforced_column_constraint_removed,
|
|
865
|
+
enforced_model_constraint_removed=enforced_model_constraint_removed,
|
|
866
|
+
breaking_changes=breaking_changes,
|
|
867
|
+
model_name=self.name,
|
|
868
|
+
model_file_path=self.original_file_path,
|
|
869
|
+
),
|
|
870
|
+
node=self,
|
|
871
|
+
)
|
|
872
|
+
else:
|
|
873
|
+
raise (
|
|
874
|
+
ContractBreakingChangeError(
|
|
875
|
+
breaking_changes=breaking_changes,
|
|
876
|
+
node=self,
|
|
877
|
+
)
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
# Otherwise, the contract has changed -- same_contract: False
|
|
881
|
+
return False
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
@dataclass
|
|
885
|
+
class SqlNode(SqlOperationResource, CompiledNode):
|
|
886
|
+
@classmethod
|
|
887
|
+
def resource_class(cls) -> Type[SqlOperationResource]:
|
|
888
|
+
return SqlOperationResource
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
# ====================================
|
|
892
|
+
# Seed node
|
|
893
|
+
# ====================================
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
@dataclass
|
|
897
|
+
class SeedNode(SeedResource, ParsedNode): # No SQLDefaults!
|
|
898
|
+
@classmethod
|
|
899
|
+
def resource_class(cls) -> Type[SeedResource]:
|
|
900
|
+
return SeedResource
|
|
901
|
+
|
|
902
|
+
def same_seeds(self, other: "SeedNode") -> bool:
|
|
903
|
+
# for seeds, we check the hashes. If the hashes are different types,
|
|
904
|
+
# no match. If the hashes are both the same 'path', log a warning and
|
|
905
|
+
# assume they are the same
|
|
906
|
+
# if the current checksum is a path, we want to log a warning.
|
|
907
|
+
result = self.checksum == other.checksum
|
|
908
|
+
|
|
909
|
+
if self.checksum.name == "path":
|
|
910
|
+
msg: str
|
|
911
|
+
if other.checksum.name != "path":
|
|
912
|
+
warn_or_error(
|
|
913
|
+
SeedIncreased(package_name=self.package_name, name=self.name), node=self
|
|
914
|
+
)
|
|
915
|
+
elif result:
|
|
916
|
+
warn_or_error(
|
|
917
|
+
SeedExceedsLimitSamePath(package_name=self.package_name, name=self.name),
|
|
918
|
+
node=self,
|
|
919
|
+
)
|
|
920
|
+
elif not result:
|
|
921
|
+
warn_or_error(
|
|
922
|
+
SeedExceedsLimitAndPathChanged(package_name=self.package_name, name=self.name),
|
|
923
|
+
node=self,
|
|
924
|
+
)
|
|
925
|
+
else:
|
|
926
|
+
warn_or_error(
|
|
927
|
+
SeedExceedsLimitChecksumChanged(
|
|
928
|
+
package_name=self.package_name,
|
|
929
|
+
name=self.name,
|
|
930
|
+
checksum_name=other.checksum.name,
|
|
931
|
+
),
|
|
932
|
+
node=self,
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
return result
|
|
936
|
+
|
|
937
|
+
@property
|
|
938
|
+
def empty(self):
|
|
939
|
+
"""Seeds are never empty"""
|
|
940
|
+
return False
|
|
941
|
+
|
|
942
|
+
def _disallow_implicit_dependencies(self):
|
|
943
|
+
"""Disallow seeds to take implicit upstream dependencies via pre/post hooks"""
|
|
944
|
+
# Seeds are root nodes in the DAG. They cannot depend on other nodes.
|
|
945
|
+
# However, it's possible to define pre- and post-hooks on seeds, and for those
|
|
946
|
+
# hooks to include {{ ref(...) }}. This worked in previous versions, but it
|
|
947
|
+
# was never officially documented or supported behavior. Let's raise an explicit error,
|
|
948
|
+
# which will surface during parsing if the user has written code such that we attempt
|
|
949
|
+
# to capture & record a ref/source/metric call on the SeedNode.
|
|
950
|
+
# For more details: https://github.com/dbt-labs/dbt-core/issues/6806
|
|
951
|
+
hooks = [f'- pre_hook: "{hook.sql}"' for hook in self.config.pre_hook] + [
|
|
952
|
+
f'- post_hook: "{hook.sql}"' for hook in self.config.post_hook
|
|
953
|
+
]
|
|
954
|
+
hook_list = "\n".join(hooks)
|
|
955
|
+
message = f"""
|
|
956
|
+
Seeds cannot depend on other nodes. dbt detected a seed with a pre- or post-hook
|
|
957
|
+
that calls 'ref', 'source', or 'metric', either directly or indirectly via other macros.
|
|
958
|
+
|
|
959
|
+
Error raised for '{self.unique_id}', which has these hooks defined: \n{hook_list}
|
|
960
|
+
"""
|
|
961
|
+
raise ParsingError(message)
|
|
962
|
+
|
|
963
|
+
@property
|
|
964
|
+
def refs(self):
|
|
965
|
+
self._disallow_implicit_dependencies()
|
|
966
|
+
|
|
967
|
+
@property
|
|
968
|
+
def sources(self):
|
|
969
|
+
self._disallow_implicit_dependencies()
|
|
970
|
+
|
|
971
|
+
@property
|
|
972
|
+
def metrics(self):
|
|
973
|
+
self._disallow_implicit_dependencies()
|
|
974
|
+
|
|
975
|
+
def same_body(self, other) -> bool:
|
|
976
|
+
return self.same_seeds(other)
|
|
977
|
+
|
|
978
|
+
@property
|
|
979
|
+
def depends_on_nodes(self):
|
|
980
|
+
return []
|
|
981
|
+
|
|
982
|
+
@property
|
|
983
|
+
def depends_on_macros(self) -> List[str]:
|
|
984
|
+
return self.depends_on.macros
|
|
985
|
+
|
|
986
|
+
@property
|
|
987
|
+
def extra_ctes(self):
|
|
988
|
+
return []
|
|
989
|
+
|
|
990
|
+
@property
|
|
991
|
+
def extra_ctes_injected(self):
|
|
992
|
+
return False
|
|
993
|
+
|
|
994
|
+
@property
|
|
995
|
+
def language(self):
|
|
996
|
+
return "sql"
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
# @property
|
|
1000
|
+
# def compiled_code(self):
|
|
1001
|
+
# return None
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
# ====================================
|
|
1005
|
+
# Singular Test node
|
|
1006
|
+
# ====================================
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
class TestShouldStoreFailures:
|
|
1010
|
+
@property
|
|
1011
|
+
def should_store_failures(self):
|
|
1012
|
+
if self.config.store_failures:
|
|
1013
|
+
return self.config.store_failures
|
|
1014
|
+
return get_flags().STORE_FAILURES
|
|
1015
|
+
|
|
1016
|
+
@property
|
|
1017
|
+
def is_relational(self):
|
|
1018
|
+
if self.should_store_failures:
|
|
1019
|
+
return True
|
|
1020
|
+
return False
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
@dataclass
|
|
1024
|
+
class SingularTestNode(SingularTestResource, TestShouldStoreFailures, CompiledNode):
|
|
1025
|
+
@classmethod
|
|
1026
|
+
def resource_class(cls) -> Type[SingularTestResource]:
|
|
1027
|
+
return SingularTestResource
|
|
1028
|
+
|
|
1029
|
+
@property
|
|
1030
|
+
def test_node_type(self):
|
|
1031
|
+
return "singular"
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
# ====================================
|
|
1035
|
+
# Generic Test node
|
|
1036
|
+
# ====================================
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
@dataclass
|
|
1040
|
+
class GenericTestNode(GenericTestResource, TestShouldStoreFailures, CompiledNode):
|
|
1041
|
+
@classmethod
|
|
1042
|
+
def resource_class(cls) -> Type[GenericTestResource]:
|
|
1043
|
+
return GenericTestResource
|
|
1044
|
+
|
|
1045
|
+
def same_contents(self, other, adapter_type: Optional[str]) -> bool:
|
|
1046
|
+
if other is None:
|
|
1047
|
+
return False
|
|
1048
|
+
|
|
1049
|
+
return self.same_config(other) and self.same_fqn(other) and True
|
|
1050
|
+
|
|
1051
|
+
@property
|
|
1052
|
+
def test_node_type(self):
|
|
1053
|
+
return "generic"
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
@dataclass
|
|
1057
|
+
class UnitTestSourceDefinition(ModelNode):
|
|
1058
|
+
source_name: str = "undefined"
|
|
1059
|
+
quoting: QuotingResource = field(default_factory=QuotingResource)
|
|
1060
|
+
|
|
1061
|
+
@property
|
|
1062
|
+
def search_name(self):
|
|
1063
|
+
return f"{self.source_name}.{self.name}"
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
@dataclass
|
|
1067
|
+
class UnitTestNode(CompiledNode):
|
|
1068
|
+
resource_type: Literal[NodeType.Unit]
|
|
1069
|
+
tested_node_unique_id: Optional[str] = None
|
|
1070
|
+
this_input_node_unique_id: Optional[str] = None
|
|
1071
|
+
overrides: Optional[UnitTestOverrides] = None
|
|
1072
|
+
config: UnitTestNodeConfig = field(default_factory=UnitTestNodeConfig)
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
@dataclass
|
|
1076
|
+
class UnitTestDefinition(NodeInfoMixin, GraphNode, UnitTestDefinitionResource):
|
|
1077
|
+
@classmethod
|
|
1078
|
+
def resource_class(cls) -> Type[UnitTestDefinitionResource]:
|
|
1079
|
+
return UnitTestDefinitionResource
|
|
1080
|
+
|
|
1081
|
+
@property
|
|
1082
|
+
def depends_on_nodes(self):
|
|
1083
|
+
return self.depends_on.nodes
|
|
1084
|
+
|
|
1085
|
+
@property
|
|
1086
|
+
def tags(self) -> List[str]:
|
|
1087
|
+
tags = self.config.tags
|
|
1088
|
+
return [tags] if isinstance(tags, str) else tags
|
|
1089
|
+
|
|
1090
|
+
@property
|
|
1091
|
+
def versioned_name(self) -> str:
|
|
1092
|
+
versioned_name = self.name
|
|
1093
|
+
if self.version is not None:
|
|
1094
|
+
versioned_name += f"_v{self.version}"
|
|
1095
|
+
return versioned_name
|
|
1096
|
+
|
|
1097
|
+
def build_unit_test_checksum(self):
|
|
1098
|
+
# everything except 'description'
|
|
1099
|
+
data = f"{self.model}-{self.versions}-{self.given}-{self.expect}-{self.overrides}"
|
|
1100
|
+
|
|
1101
|
+
# include underlying fixture data
|
|
1102
|
+
for input in self.given:
|
|
1103
|
+
if input.fixture:
|
|
1104
|
+
data += f"-{input.rows}"
|
|
1105
|
+
|
|
1106
|
+
self.checksum = hashlib.new("sha256", data.encode("utf-8")).hexdigest()
|
|
1107
|
+
|
|
1108
|
+
def same_contents(self, other: Optional["UnitTestDefinition"]) -> bool:
|
|
1109
|
+
if other is None:
|
|
1110
|
+
return False
|
|
1111
|
+
|
|
1112
|
+
return self.checksum == other.checksum
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
@dataclass
|
|
1116
|
+
class UnitTestFileFixture(BaseNode):
|
|
1117
|
+
resource_type: Literal[NodeType.Fixture]
|
|
1118
|
+
rows: Optional[Union[List[Dict[str, Any]], str]] = None
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
# ====================================
|
|
1122
|
+
# Snapshot node
|
|
1123
|
+
# ====================================
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
@dataclass
|
|
1127
|
+
class SnapshotNode(SnapshotResource, CompiledNode):
|
|
1128
|
+
@classmethod
|
|
1129
|
+
def resource_class(cls) -> Type[SnapshotResource]:
|
|
1130
|
+
return SnapshotResource
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
# ====================================
|
|
1134
|
+
# Macro
|
|
1135
|
+
# ====================================
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
@dataclass
|
|
1139
|
+
class Macro(MacroResource, BaseNode):
|
|
1140
|
+
@classmethod
|
|
1141
|
+
def resource_class(cls) -> Type[MacroResource]:
|
|
1142
|
+
return MacroResource
|
|
1143
|
+
|
|
1144
|
+
def same_contents(self, other: Optional["Macro"]) -> bool:
|
|
1145
|
+
if other is None:
|
|
1146
|
+
return False
|
|
1147
|
+
# the only thing that makes one macro different from another with the
|
|
1148
|
+
# same name/package is its content
|
|
1149
|
+
return self.macro_sql == other.macro_sql
|
|
1150
|
+
|
|
1151
|
+
@property
|
|
1152
|
+
def depends_on_macros(self):
|
|
1153
|
+
return self.depends_on.macros
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
# ====================================
|
|
1157
|
+
# Documentation node
|
|
1158
|
+
# ====================================
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
@dataclass
|
|
1162
|
+
class Documentation(DocumentationResource, BaseNode):
|
|
1163
|
+
@classmethod
|
|
1164
|
+
def resource_class(cls) -> Type[DocumentationResource]:
|
|
1165
|
+
return DocumentationResource
|
|
1166
|
+
|
|
1167
|
+
@property
|
|
1168
|
+
def search_name(self):
|
|
1169
|
+
return self.name
|
|
1170
|
+
|
|
1171
|
+
def same_contents(self, other: Optional["Documentation"]) -> bool:
|
|
1172
|
+
if other is None:
|
|
1173
|
+
return False
|
|
1174
|
+
# the only thing that makes one doc different from another with the
|
|
1175
|
+
# same name/package is its content
|
|
1176
|
+
return self.block_contents == other.block_contents
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
# ====================================
|
|
1180
|
+
# Source node
|
|
1181
|
+
# ====================================
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
def normalize_test(testdef: TestDef) -> Dict[str, Any]:
|
|
1185
|
+
if isinstance(testdef, str):
|
|
1186
|
+
return {testdef: {}}
|
|
1187
|
+
else:
|
|
1188
|
+
return testdef
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
@dataclass
|
|
1192
|
+
class UnpatchedSourceDefinition(BaseNode):
|
|
1193
|
+
source: UnparsedSourceDefinition
|
|
1194
|
+
table: UnparsedSourceTableDefinition
|
|
1195
|
+
fqn: List[str]
|
|
1196
|
+
resource_type: Literal[NodeType.Source]
|
|
1197
|
+
patch_path: Optional[str] = None
|
|
1198
|
+
|
|
1199
|
+
def get_full_source_name(self):
|
|
1200
|
+
return f"{self.source.name}_{self.table.name}"
|
|
1201
|
+
|
|
1202
|
+
def get_source_representation(self):
|
|
1203
|
+
return f'source("{self.source.name}", "{self.table.name}")'
|
|
1204
|
+
|
|
1205
|
+
def validate_data_tests(self, is_root_project: bool):
|
|
1206
|
+
"""
|
|
1207
|
+
sources parse tests differently than models, so we need to do some validation
|
|
1208
|
+
here where it's done in the PatchParser for other nodes
|
|
1209
|
+
"""
|
|
1210
|
+
# source table-level tests
|
|
1211
|
+
if self.tests and self.data_tests:
|
|
1212
|
+
raise ValidationError(
|
|
1213
|
+
"Invalid test config: cannot have both 'tests' and 'data_tests' defined"
|
|
1214
|
+
)
|
|
1215
|
+
if self.tests:
|
|
1216
|
+
self.data_tests.extend(self.tests)
|
|
1217
|
+
self.tests.clear()
|
|
1218
|
+
|
|
1219
|
+
# column-level tests
|
|
1220
|
+
for column in self.columns:
|
|
1221
|
+
if column.tests and column.data_tests:
|
|
1222
|
+
raise ValidationError(
|
|
1223
|
+
"Invalid test config: cannot have both 'tests' and 'data_tests' defined"
|
|
1224
|
+
)
|
|
1225
|
+
if column.tests:
|
|
1226
|
+
column.data_tests.extend(column.tests)
|
|
1227
|
+
column.tests.clear()
|
|
1228
|
+
|
|
1229
|
+
@property
|
|
1230
|
+
def quote_columns(self) -> Optional[bool]:
|
|
1231
|
+
result = None
|
|
1232
|
+
if self.source.quoting.column is not None:
|
|
1233
|
+
result = self.source.quoting.column
|
|
1234
|
+
if self.table.quoting.column is not None:
|
|
1235
|
+
result = self.table.quoting.column
|
|
1236
|
+
return result
|
|
1237
|
+
|
|
1238
|
+
@property
|
|
1239
|
+
def columns(self) -> Sequence[UnparsedColumn]:
|
|
1240
|
+
return [] if self.table.columns is None else self.table.columns
|
|
1241
|
+
|
|
1242
|
+
def get_tests(self) -> Iterator[Tuple[Dict[str, Any], Optional[UnparsedColumn]]]:
|
|
1243
|
+
for data_test in self.data_tests:
|
|
1244
|
+
yield normalize_test(data_test), None
|
|
1245
|
+
|
|
1246
|
+
for column in self.columns:
|
|
1247
|
+
if column.data_tests is not None:
|
|
1248
|
+
for data_test in column.data_tests:
|
|
1249
|
+
yield normalize_test(data_test), column
|
|
1250
|
+
|
|
1251
|
+
@property
|
|
1252
|
+
def data_tests(self) -> List[TestDef]:
|
|
1253
|
+
if self.table.data_tests is None:
|
|
1254
|
+
return []
|
|
1255
|
+
else:
|
|
1256
|
+
return self.table.data_tests
|
|
1257
|
+
|
|
1258
|
+
# deprecated
|
|
1259
|
+
@property
|
|
1260
|
+
def tests(self) -> List[TestDef]:
|
|
1261
|
+
if self.table.tests is None:
|
|
1262
|
+
return []
|
|
1263
|
+
else:
|
|
1264
|
+
return self.table.tests
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
@dataclass
|
|
1268
|
+
class SourceDefinition(
|
|
1269
|
+
NodeInfoMixin,
|
|
1270
|
+
GraphNode,
|
|
1271
|
+
SourceDefinitionResource,
|
|
1272
|
+
HasRelationMetadata,
|
|
1273
|
+
):
|
|
1274
|
+
@classmethod
|
|
1275
|
+
def resource_class(cls) -> Type[SourceDefinitionResource]:
|
|
1276
|
+
return SourceDefinitionResource
|
|
1277
|
+
|
|
1278
|
+
def same_database_representation(self, other: "SourceDefinition") -> bool:
|
|
1279
|
+
|
|
1280
|
+
# preserve legacy behaviour -- use potentially rendered database
|
|
1281
|
+
if get_flags().state_modified_compare_more_unrendered_values is False:
|
|
1282
|
+
same_database = self.database == other.database
|
|
1283
|
+
same_schema = self.schema == other.schema
|
|
1284
|
+
else:
|
|
1285
|
+
same_database = self.unrendered_database == other.unrendered_database
|
|
1286
|
+
same_schema = self.unrendered_schema == other.unrendered_schema
|
|
1287
|
+
|
|
1288
|
+
return same_database and same_schema and self.identifier == other.identifier and True
|
|
1289
|
+
|
|
1290
|
+
def same_quoting(self, other: "SourceDefinition") -> bool:
|
|
1291
|
+
return self.quoting == other.quoting
|
|
1292
|
+
|
|
1293
|
+
def same_freshness(self, other: "SourceDefinition") -> bool:
|
|
1294
|
+
return (
|
|
1295
|
+
self.freshness == other.freshness
|
|
1296
|
+
and self.loaded_at_field == other.loaded_at_field
|
|
1297
|
+
and True
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
def same_external(self, other: "SourceDefinition") -> bool:
|
|
1301
|
+
return self.external == other.external
|
|
1302
|
+
|
|
1303
|
+
def same_config(self, old: "SourceDefinition") -> bool:
|
|
1304
|
+
return self.config.same_contents(
|
|
1305
|
+
self.unrendered_config,
|
|
1306
|
+
old.unrendered_config,
|
|
1307
|
+
)
|
|
1308
|
+
|
|
1309
|
+
def same_contents(self, old: Optional["SourceDefinition"]) -> bool:
|
|
1310
|
+
# existing when it didn't before is a change!
|
|
1311
|
+
if old is None:
|
|
1312
|
+
return True
|
|
1313
|
+
|
|
1314
|
+
# config changes are changes (because the only config is "enforced", and
|
|
1315
|
+
# enabling a source is a change!)
|
|
1316
|
+
# changing the database/schema/identifier is a change
|
|
1317
|
+
# messing around with external stuff is a change (uh, right?)
|
|
1318
|
+
# quoting changes are changes
|
|
1319
|
+
# freshness changes are changes, I guess
|
|
1320
|
+
# metadata/tags changes are not "changes"
|
|
1321
|
+
# patching/description changes are not "changes"
|
|
1322
|
+
return (
|
|
1323
|
+
self.same_database_representation(old)
|
|
1324
|
+
and self.same_fqn(old)
|
|
1325
|
+
and self.same_config(old)
|
|
1326
|
+
and self.same_quoting(old)
|
|
1327
|
+
and self.same_freshness(old)
|
|
1328
|
+
and self.same_external(old)
|
|
1329
|
+
and True
|
|
1330
|
+
)
|
|
1331
|
+
|
|
1332
|
+
def get_full_source_name(self):
|
|
1333
|
+
return f"{self.source_name}_{self.name}"
|
|
1334
|
+
|
|
1335
|
+
def get_source_representation(self):
|
|
1336
|
+
return f'source("{self.source.name}", "{self.table.name}")'
|
|
1337
|
+
|
|
1338
|
+
@property
|
|
1339
|
+
def is_refable(self):
|
|
1340
|
+
return False
|
|
1341
|
+
|
|
1342
|
+
@property
|
|
1343
|
+
def is_ephemeral(self):
|
|
1344
|
+
return False
|
|
1345
|
+
|
|
1346
|
+
@property
|
|
1347
|
+
def is_ephemeral_model(self):
|
|
1348
|
+
return False
|
|
1349
|
+
|
|
1350
|
+
@property
|
|
1351
|
+
def depends_on_nodes(self):
|
|
1352
|
+
return []
|
|
1353
|
+
|
|
1354
|
+
@property
|
|
1355
|
+
def depends_on(self):
|
|
1356
|
+
return DependsOn(macros=[], nodes=[])
|
|
1357
|
+
|
|
1358
|
+
@property
|
|
1359
|
+
def refs(self):
|
|
1360
|
+
return []
|
|
1361
|
+
|
|
1362
|
+
@property
|
|
1363
|
+
def sources(self):
|
|
1364
|
+
return []
|
|
1365
|
+
|
|
1366
|
+
@property
|
|
1367
|
+
def has_freshness(self) -> bool:
|
|
1368
|
+
return bool(self.freshness)
|
|
1369
|
+
|
|
1370
|
+
@property
|
|
1371
|
+
def search_name(self):
|
|
1372
|
+
return f"{self.source_name}.{self.name}"
|
|
1373
|
+
|
|
1374
|
+
@property
|
|
1375
|
+
def group(self):
|
|
1376
|
+
return None
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
# ====================================
|
|
1380
|
+
# Exposure node
|
|
1381
|
+
# ====================================
|
|
1382
|
+
|
|
1383
|
+
|
|
1384
|
+
@dataclass
|
|
1385
|
+
class Exposure(NodeInfoMixin, GraphNode, ExposureResource):
|
|
1386
|
+
@property
|
|
1387
|
+
def depends_on_nodes(self):
|
|
1388
|
+
return self.depends_on.nodes
|
|
1389
|
+
|
|
1390
|
+
@property
|
|
1391
|
+
def search_name(self):
|
|
1392
|
+
return self.name
|
|
1393
|
+
|
|
1394
|
+
@classmethod
|
|
1395
|
+
def resource_class(cls) -> Type[ExposureResource]:
|
|
1396
|
+
return ExposureResource
|
|
1397
|
+
|
|
1398
|
+
def same_depends_on(self, old: "Exposure") -> bool:
|
|
1399
|
+
return set(self.depends_on.nodes) == set(old.depends_on.nodes)
|
|
1400
|
+
|
|
1401
|
+
def same_description(self, old: "Exposure") -> bool:
|
|
1402
|
+
return self.description == old.description
|
|
1403
|
+
|
|
1404
|
+
def same_label(self, old: "Exposure") -> bool:
|
|
1405
|
+
return self.label == old.label
|
|
1406
|
+
|
|
1407
|
+
def same_maturity(self, old: "Exposure") -> bool:
|
|
1408
|
+
return self.maturity == old.maturity
|
|
1409
|
+
|
|
1410
|
+
def same_owner(self, old: "Exposure") -> bool:
|
|
1411
|
+
return self.owner == old.owner
|
|
1412
|
+
|
|
1413
|
+
def same_exposure_type(self, old: "Exposure") -> bool:
|
|
1414
|
+
return self.type == old.type
|
|
1415
|
+
|
|
1416
|
+
def same_url(self, old: "Exposure") -> bool:
|
|
1417
|
+
return self.url == old.url
|
|
1418
|
+
|
|
1419
|
+
def same_config(self, old: "Exposure") -> bool:
|
|
1420
|
+
return self.config.same_contents(
|
|
1421
|
+
self.unrendered_config,
|
|
1422
|
+
old.unrendered_config,
|
|
1423
|
+
)
|
|
1424
|
+
|
|
1425
|
+
def same_contents(self, old: Optional["Exposure"]) -> bool:
|
|
1426
|
+
# existing when it didn't before is a change!
|
|
1427
|
+
# metadata/tags changes are not "changes"
|
|
1428
|
+
if old is None:
|
|
1429
|
+
return True
|
|
1430
|
+
|
|
1431
|
+
return (
|
|
1432
|
+
self.same_fqn(old)
|
|
1433
|
+
and self.same_exposure_type(old)
|
|
1434
|
+
and self.same_owner(old)
|
|
1435
|
+
and self.same_maturity(old)
|
|
1436
|
+
and self.same_url(old)
|
|
1437
|
+
and self.same_description(old)
|
|
1438
|
+
and self.same_label(old)
|
|
1439
|
+
and self.same_depends_on(old)
|
|
1440
|
+
and self.same_config(old)
|
|
1441
|
+
and True
|
|
1442
|
+
)
|
|
1443
|
+
|
|
1444
|
+
@property
|
|
1445
|
+
def group(self):
|
|
1446
|
+
return None
|
|
1447
|
+
|
|
1448
|
+
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
|
1449
|
+
dct = super().__post_serialize__(dct, context)
|
|
1450
|
+
if "_event_status" in dct:
|
|
1451
|
+
del dct["_event_status"]
|
|
1452
|
+
return dct
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
# ====================================
|
|
1456
|
+
# Metric node
|
|
1457
|
+
# ====================================
|
|
1458
|
+
|
|
1459
|
+
|
|
1460
|
+
@dataclass
|
|
1461
|
+
class Metric(GraphNode, MetricResource):
|
|
1462
|
+
@property
|
|
1463
|
+
def depends_on_nodes(self):
|
|
1464
|
+
return self.depends_on.nodes
|
|
1465
|
+
|
|
1466
|
+
@property
|
|
1467
|
+
def search_name(self):
|
|
1468
|
+
return self.name
|
|
1469
|
+
|
|
1470
|
+
@classmethod
|
|
1471
|
+
def resource_class(cls) -> Type[MetricResource]:
|
|
1472
|
+
return MetricResource
|
|
1473
|
+
|
|
1474
|
+
def same_description(self, old: "Metric") -> bool:
|
|
1475
|
+
return self.description == old.description
|
|
1476
|
+
|
|
1477
|
+
def same_label(self, old: "Metric") -> bool:
|
|
1478
|
+
return self.label == old.label
|
|
1479
|
+
|
|
1480
|
+
def same_config(self, old: "Metric") -> bool:
|
|
1481
|
+
return self.config.same_contents(
|
|
1482
|
+
self.unrendered_config,
|
|
1483
|
+
old.unrendered_config,
|
|
1484
|
+
)
|
|
1485
|
+
|
|
1486
|
+
def same_filter(self, old: "Metric") -> bool:
|
|
1487
|
+
return True # TODO
|
|
1488
|
+
|
|
1489
|
+
def same_metadata(self, old: "Metric") -> bool:
|
|
1490
|
+
return True # TODO
|
|
1491
|
+
|
|
1492
|
+
def same_type(self, old: "Metric") -> bool:
|
|
1493
|
+
return self.type == old.type
|
|
1494
|
+
|
|
1495
|
+
def same_type_params(self, old: "Metric") -> bool:
|
|
1496
|
+
return True # TODO
|
|
1497
|
+
|
|
1498
|
+
def same_contents(self, old: Optional["Metric"]) -> bool:
|
|
1499
|
+
# existing when it didn't before is a change!
|
|
1500
|
+
# metadata/tags changes are not "changes"
|
|
1501
|
+
if old is None:
|
|
1502
|
+
return True
|
|
1503
|
+
|
|
1504
|
+
return (
|
|
1505
|
+
self.same_filter(old)
|
|
1506
|
+
and self.same_metadata(old)
|
|
1507
|
+
and self.same_type(old)
|
|
1508
|
+
and self.same_type_params(old)
|
|
1509
|
+
and self.same_description(old)
|
|
1510
|
+
and self.same_label(old)
|
|
1511
|
+
and self.same_config(old)
|
|
1512
|
+
and True
|
|
1513
|
+
)
|
|
1514
|
+
|
|
1515
|
+
def add_input_measure(self, input_measure: MetricInputMeasure) -> None:
|
|
1516
|
+
for existing_input_measure in self.type_params.input_measures:
|
|
1517
|
+
if input_measure == existing_input_measure:
|
|
1518
|
+
return
|
|
1519
|
+
self.type_params.input_measures.append(input_measure)
|
|
1520
|
+
|
|
1521
|
+
|
|
1522
|
+
# ====================================
|
|
1523
|
+
# Group node
|
|
1524
|
+
# ====================================
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
@dataclass
|
|
1528
|
+
class Group(GroupResource, BaseNode):
|
|
1529
|
+
@classmethod
|
|
1530
|
+
def resource_class(cls) -> Type[GroupResource]:
|
|
1531
|
+
return GroupResource
|
|
1532
|
+
|
|
1533
|
+
def to_logging_dict(self) -> Dict[str, Union[str, Dict[str, str]]]:
|
|
1534
|
+
return {
|
|
1535
|
+
"name": self.name,
|
|
1536
|
+
"package_name": self.package_name,
|
|
1537
|
+
"owner": {k: str(v) for k, v in self.owner.to_dict(omit_none=True).items()},
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
# ====================================
|
|
1542
|
+
# Function node
|
|
1543
|
+
# ====================================
|
|
1544
|
+
|
|
1545
|
+
|
|
1546
|
+
@dataclass
|
|
1547
|
+
class FunctionNode(CompiledNode, FunctionResource):
|
|
1548
|
+
|
|
1549
|
+
@classmethod
|
|
1550
|
+
def resource_class(cls) -> Type[FunctionResource]:
|
|
1551
|
+
return FunctionResource
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
# ====================================
|
|
1555
|
+
# SemanticModel node
|
|
1556
|
+
# ====================================
|
|
1557
|
+
|
|
1558
|
+
|
|
1559
|
+
@dataclass
|
|
1560
|
+
class SemanticModel(GraphNode, SemanticModelResource):
|
|
1561
|
+
@property
|
|
1562
|
+
def depends_on_nodes(self):
|
|
1563
|
+
return self.depends_on.nodes
|
|
1564
|
+
|
|
1565
|
+
@property
|
|
1566
|
+
def depends_on_macros(self):
|
|
1567
|
+
return self.depends_on.macros
|
|
1568
|
+
|
|
1569
|
+
@classmethod
|
|
1570
|
+
def resource_class(cls) -> Type[SemanticModelResource]:
|
|
1571
|
+
return SemanticModelResource
|
|
1572
|
+
|
|
1573
|
+
def same_model(self, old: "SemanticModel") -> bool:
|
|
1574
|
+
return self.model == old.model
|
|
1575
|
+
|
|
1576
|
+
def same_description(self, old: "SemanticModel") -> bool:
|
|
1577
|
+
return self.description == old.description
|
|
1578
|
+
|
|
1579
|
+
def same_defaults(self, old: "SemanticModel") -> bool:
|
|
1580
|
+
return self.defaults == old.defaults
|
|
1581
|
+
|
|
1582
|
+
def same_entities(self, old: "SemanticModel") -> bool:
|
|
1583
|
+
return self.entities == old.entities
|
|
1584
|
+
|
|
1585
|
+
def same_dimensions(self, old: "SemanticModel") -> bool:
|
|
1586
|
+
return self.dimensions == old.dimensions
|
|
1587
|
+
|
|
1588
|
+
def same_measures(self, old: "SemanticModel") -> bool:
|
|
1589
|
+
return self.measures == old.measures
|
|
1590
|
+
|
|
1591
|
+
def same_config(self, old: "SemanticModel") -> bool:
|
|
1592
|
+
return self.config == old.config
|
|
1593
|
+
|
|
1594
|
+
def same_primary_entity(self, old: "SemanticModel") -> bool:
|
|
1595
|
+
return self.primary_entity == old.primary_entity
|
|
1596
|
+
|
|
1597
|
+
def same_group(self, old: "SemanticModel") -> bool:
|
|
1598
|
+
return self.group == old.group
|
|
1599
|
+
|
|
1600
|
+
def same_contents(self, old: Optional["SemanticModel"]) -> bool:
|
|
1601
|
+
# existing when it didn't before is a change!
|
|
1602
|
+
# metadata/tags changes are not "changes"
|
|
1603
|
+
if old is None:
|
|
1604
|
+
return True
|
|
1605
|
+
|
|
1606
|
+
return (
|
|
1607
|
+
self.same_model(old)
|
|
1608
|
+
and self.same_description(old)
|
|
1609
|
+
and self.same_defaults(old)
|
|
1610
|
+
and self.same_entities(old)
|
|
1611
|
+
and self.same_dimensions(old)
|
|
1612
|
+
and self.same_measures(old)
|
|
1613
|
+
and self.same_config(old)
|
|
1614
|
+
and self.same_primary_entity(old)
|
|
1615
|
+
and self.same_group(old)
|
|
1616
|
+
and True
|
|
1617
|
+
)
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
# ====================================
|
|
1621
|
+
# SavedQuery
|
|
1622
|
+
# ====================================
|
|
1623
|
+
|
|
1624
|
+
|
|
1625
|
+
@dataclass
|
|
1626
|
+
class SavedQuery(NodeInfoMixin, GraphNode, SavedQueryResource):
|
|
1627
|
+
@classmethod
|
|
1628
|
+
def resource_class(cls) -> Type[SavedQueryResource]:
|
|
1629
|
+
return SavedQueryResource
|
|
1630
|
+
|
|
1631
|
+
def same_metrics(self, old: "SavedQuery") -> bool:
|
|
1632
|
+
return self.query_params.metrics == old.query_params.metrics
|
|
1633
|
+
|
|
1634
|
+
def same_group_by(self, old: "SavedQuery") -> bool:
|
|
1635
|
+
return self.query_params.group_by == old.query_params.group_by
|
|
1636
|
+
|
|
1637
|
+
def same_description(self, old: "SavedQuery") -> bool:
|
|
1638
|
+
return self.description == old.description
|
|
1639
|
+
|
|
1640
|
+
def same_where(self, old: "SavedQuery") -> bool:
|
|
1641
|
+
return self.query_params.where == old.query_params.where
|
|
1642
|
+
|
|
1643
|
+
def same_label(self, old: "SavedQuery") -> bool:
|
|
1644
|
+
return self.label == old.label
|
|
1645
|
+
|
|
1646
|
+
def same_config(self, old: "SavedQuery") -> bool:
|
|
1647
|
+
return self.config == old.config
|
|
1648
|
+
|
|
1649
|
+
def same_group(self, old: "SavedQuery") -> bool:
|
|
1650
|
+
return self.group == old.group
|
|
1651
|
+
|
|
1652
|
+
def same_exports(self, old: "SavedQuery") -> bool:
|
|
1653
|
+
if len(self.exports) != len(old.exports):
|
|
1654
|
+
return False
|
|
1655
|
+
|
|
1656
|
+
# exports should be in the same order, so we zip them for easy iteration
|
|
1657
|
+
for old_export, new_export in zip(old.exports, self.exports):
|
|
1658
|
+
if not (old_export.name == new_export.name):
|
|
1659
|
+
return False
|
|
1660
|
+
keys = ["export_as", "schema", "alias"]
|
|
1661
|
+
for key in keys:
|
|
1662
|
+
if old_export.unrendered_config.get(key) != new_export.unrendered_config.get(key):
|
|
1663
|
+
return False
|
|
1664
|
+
|
|
1665
|
+
return True
|
|
1666
|
+
|
|
1667
|
+
def same_tags(self, old: "SavedQuery") -> bool:
|
|
1668
|
+
return self.tags == old.tags
|
|
1669
|
+
|
|
1670
|
+
def same_contents(self, old: Optional["SavedQuery"]) -> bool:
|
|
1671
|
+
# existing when it didn't before is a change!
|
|
1672
|
+
# metadata/tags changes are not "changes"
|
|
1673
|
+
if old is None:
|
|
1674
|
+
return True
|
|
1675
|
+
|
|
1676
|
+
return (
|
|
1677
|
+
self.same_metrics(old)
|
|
1678
|
+
and self.same_group_by(old)
|
|
1679
|
+
and self.same_description(old)
|
|
1680
|
+
and self.same_where(old)
|
|
1681
|
+
and self.same_label(old)
|
|
1682
|
+
and self.same_config(old)
|
|
1683
|
+
and self.same_group(old)
|
|
1684
|
+
and self.same_exports(old)
|
|
1685
|
+
and self.same_tags(old)
|
|
1686
|
+
and True
|
|
1687
|
+
)
|
|
1688
|
+
|
|
1689
|
+
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
|
1690
|
+
dct = super().__post_serialize__(dct, context)
|
|
1691
|
+
if "_event_status" in dct:
|
|
1692
|
+
del dct["_event_status"]
|
|
1693
|
+
return dct
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
# ====================================
|
|
1697
|
+
# Patches
|
|
1698
|
+
# ====================================
|
|
1699
|
+
|
|
1700
|
+
|
|
1701
|
+
@dataclass
|
|
1702
|
+
class ParsedPatch(HasYamlMetadata):
|
|
1703
|
+
name: str
|
|
1704
|
+
description: str
|
|
1705
|
+
meta: Dict[str, Any]
|
|
1706
|
+
docs: Docs
|
|
1707
|
+
config: Dict[str, Any]
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
# The parsed node update is only the 'patch', not the test. The test became a
|
|
1711
|
+
# regular parsed node. Note that description and columns must be present, but
|
|
1712
|
+
# may be empty.
|
|
1713
|
+
@dataclass
|
|
1714
|
+
class ParsedNodePatch(ParsedPatch):
|
|
1715
|
+
columns: Dict[str, ColumnInfo]
|
|
1716
|
+
access: Optional[str]
|
|
1717
|
+
version: Optional[NodeVersion]
|
|
1718
|
+
latest_version: Optional[NodeVersion]
|
|
1719
|
+
constraints: List[Dict[str, Any]]
|
|
1720
|
+
deprecation_date: Optional[datetime]
|
|
1721
|
+
time_spine: Optional[TimeSpine] = None
|
|
1722
|
+
freshness: Optional[ModelFreshness] = None
|
|
1723
|
+
|
|
1724
|
+
|
|
1725
|
+
@dataclass
|
|
1726
|
+
class ParsedFunctionPatchRequired:
|
|
1727
|
+
returns: FunctionReturns
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
# TODO: Maybe this shouldn't be a subclass of ParsedNodePatch, but ParsedPatch instead
|
|
1731
|
+
# Currently, `functions` have the fields like `columns`, `access`, `version`, and etc,
|
|
1732
|
+
# but they don't actually do anything. If we remove those properties from FunctionNode,
|
|
1733
|
+
# we can remove this class and use ParsedPatch instead.
|
|
1734
|
+
@dataclass
|
|
1735
|
+
class ParsedFunctionPatch(ParsedNodePatch, ParsedFunctionPatchRequired):
|
|
1736
|
+
arguments: List[FunctionArgument] = field(default_factory=list)
|
|
1737
|
+
|
|
1738
|
+
|
|
1739
|
+
@dataclass
|
|
1740
|
+
class ParsedMacroPatch(ParsedPatch):
|
|
1741
|
+
arguments: List[MacroArgument] = field(default_factory=list)
|
|
1742
|
+
|
|
1743
|
+
|
|
1744
|
+
@dataclass
|
|
1745
|
+
class ParsedSingularTestPatch(ParsedPatch):
|
|
1746
|
+
pass
|
|
1747
|
+
|
|
1748
|
+
|
|
1749
|
+
# ====================================
|
|
1750
|
+
# Node unions/categories
|
|
1751
|
+
# ====================================
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
# ManifestNode without SeedNode, which doesn't have the
|
|
1755
|
+
# SQL related attributes
|
|
1756
|
+
ManifestSQLNode = Union[
|
|
1757
|
+
AnalysisNode,
|
|
1758
|
+
FunctionNode,
|
|
1759
|
+
SingularTestNode,
|
|
1760
|
+
HookNode,
|
|
1761
|
+
ModelNode,
|
|
1762
|
+
SqlNode,
|
|
1763
|
+
GenericTestNode,
|
|
1764
|
+
SnapshotNode,
|
|
1765
|
+
UnitTestNode,
|
|
1766
|
+
]
|
|
1767
|
+
|
|
1768
|
+
# All SQL nodes plus SeedNode (csv files)
|
|
1769
|
+
ManifestNode = Union[
|
|
1770
|
+
ManifestSQLNode,
|
|
1771
|
+
SeedNode,
|
|
1772
|
+
]
|
|
1773
|
+
|
|
1774
|
+
ResultNode = Union[
|
|
1775
|
+
ManifestNode,
|
|
1776
|
+
SourceDefinition,
|
|
1777
|
+
HookNode,
|
|
1778
|
+
]
|
|
1779
|
+
|
|
1780
|
+
# All nodes that can be in the DAG
|
|
1781
|
+
GraphMemberNode = Union[
|
|
1782
|
+
ResultNode,
|
|
1783
|
+
Exposure,
|
|
1784
|
+
Metric,
|
|
1785
|
+
SavedQuery,
|
|
1786
|
+
SemanticModel,
|
|
1787
|
+
UnitTestDefinition,
|
|
1788
|
+
]
|
|
1789
|
+
|
|
1790
|
+
# All "nodes" (or node-like objects) in this file
|
|
1791
|
+
Resource = Union[
|
|
1792
|
+
GraphMemberNode,
|
|
1793
|
+
Documentation,
|
|
1794
|
+
Macro,
|
|
1795
|
+
Group,
|
|
1796
|
+
]
|
|
1797
|
+
|
|
1798
|
+
TestNode = Union[SingularTestNode, GenericTestNode]
|
|
1799
|
+
|
|
1800
|
+
SemanticManifestNode = Union[SavedQuery, SemanticModel, Metric]
|
|
1801
|
+
|
|
1802
|
+
RESOURCE_CLASS_TO_NODE_CLASS: Dict[Type[BaseResource], Type[BaseNode]] = {
|
|
1803
|
+
node_class.resource_class(): node_class
|
|
1804
|
+
for node_class in get_args(Resource)
|
|
1805
|
+
if node_class is not UnitTestNode
|
|
1806
|
+
}
|