dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dvt-core might be problematic. Click here for more details.

Files changed (275) hide show
  1. dbt/__init__.py +7 -0
  2. dbt/_pydantic_shim.py +26 -0
  3. dbt/artifacts/__init__.py +0 -0
  4. dbt/artifacts/exceptions/__init__.py +1 -0
  5. dbt/artifacts/exceptions/schemas.py +31 -0
  6. dbt/artifacts/resources/__init__.py +116 -0
  7. dbt/artifacts/resources/base.py +67 -0
  8. dbt/artifacts/resources/types.py +93 -0
  9. dbt/artifacts/resources/v1/analysis.py +10 -0
  10. dbt/artifacts/resources/v1/catalog.py +23 -0
  11. dbt/artifacts/resources/v1/components.py +274 -0
  12. dbt/artifacts/resources/v1/config.py +277 -0
  13. dbt/artifacts/resources/v1/documentation.py +11 -0
  14. dbt/artifacts/resources/v1/exposure.py +51 -0
  15. dbt/artifacts/resources/v1/function.py +52 -0
  16. dbt/artifacts/resources/v1/generic_test.py +31 -0
  17. dbt/artifacts/resources/v1/group.py +21 -0
  18. dbt/artifacts/resources/v1/hook.py +11 -0
  19. dbt/artifacts/resources/v1/macro.py +29 -0
  20. dbt/artifacts/resources/v1/metric.py +172 -0
  21. dbt/artifacts/resources/v1/model.py +145 -0
  22. dbt/artifacts/resources/v1/owner.py +10 -0
  23. dbt/artifacts/resources/v1/saved_query.py +111 -0
  24. dbt/artifacts/resources/v1/seed.py +41 -0
  25. dbt/artifacts/resources/v1/semantic_layer_components.py +72 -0
  26. dbt/artifacts/resources/v1/semantic_model.py +314 -0
  27. dbt/artifacts/resources/v1/singular_test.py +14 -0
  28. dbt/artifacts/resources/v1/snapshot.py +91 -0
  29. dbt/artifacts/resources/v1/source_definition.py +84 -0
  30. dbt/artifacts/resources/v1/sql_operation.py +10 -0
  31. dbt/artifacts/resources/v1/unit_test_definition.py +77 -0
  32. dbt/artifacts/schemas/__init__.py +0 -0
  33. dbt/artifacts/schemas/base.py +191 -0
  34. dbt/artifacts/schemas/batch_results.py +24 -0
  35. dbt/artifacts/schemas/catalog/__init__.py +11 -0
  36. dbt/artifacts/schemas/catalog/v1/__init__.py +0 -0
  37. dbt/artifacts/schemas/catalog/v1/catalog.py +59 -0
  38. dbt/artifacts/schemas/freshness/__init__.py +1 -0
  39. dbt/artifacts/schemas/freshness/v3/__init__.py +0 -0
  40. dbt/artifacts/schemas/freshness/v3/freshness.py +158 -0
  41. dbt/artifacts/schemas/manifest/__init__.py +2 -0
  42. dbt/artifacts/schemas/manifest/v12/__init__.py +0 -0
  43. dbt/artifacts/schemas/manifest/v12/manifest.py +211 -0
  44. dbt/artifacts/schemas/results.py +147 -0
  45. dbt/artifacts/schemas/run/__init__.py +2 -0
  46. dbt/artifacts/schemas/run/v5/__init__.py +0 -0
  47. dbt/artifacts/schemas/run/v5/run.py +184 -0
  48. dbt/artifacts/schemas/upgrades/__init__.py +4 -0
  49. dbt/artifacts/schemas/upgrades/upgrade_manifest.py +174 -0
  50. dbt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py +2 -0
  51. dbt/artifacts/utils/validation.py +153 -0
  52. dbt/cli/__init__.py +1 -0
  53. dbt/cli/context.py +17 -0
  54. dbt/cli/exceptions.py +57 -0
  55. dbt/cli/flags.py +560 -0
  56. dbt/cli/main.py +2039 -0
  57. dbt/cli/option_types.py +121 -0
  58. dbt/cli/options.py +80 -0
  59. dbt/cli/params.py +804 -0
  60. dbt/cli/requires.py +490 -0
  61. dbt/cli/resolvers.py +50 -0
  62. dbt/cli/types.py +40 -0
  63. dbt/clients/__init__.py +0 -0
  64. dbt/clients/checked_load.py +83 -0
  65. dbt/clients/git.py +164 -0
  66. dbt/clients/jinja.py +206 -0
  67. dbt/clients/jinja_static.py +245 -0
  68. dbt/clients/registry.py +192 -0
  69. dbt/clients/yaml_helper.py +68 -0
  70. dbt/compilation.py +876 -0
  71. dbt/compute/__init__.py +14 -0
  72. dbt/compute/engines/__init__.py +12 -0
  73. dbt/compute/engines/spark_engine.py +624 -0
  74. dbt/compute/federated_executor.py +837 -0
  75. dbt/compute/filter_pushdown.cpython-310-darwin.so +0 -0
  76. dbt/compute/filter_pushdown.py +273 -0
  77. dbt/compute/jar_provisioning.cpython-310-darwin.so +0 -0
  78. dbt/compute/jar_provisioning.py +255 -0
  79. dbt/compute/java_compat.cpython-310-darwin.so +0 -0
  80. dbt/compute/java_compat.py +689 -0
  81. dbt/compute/jdbc_utils.cpython-310-darwin.so +0 -0
  82. dbt/compute/jdbc_utils.py +678 -0
  83. dbt/compute/smart_selector.cpython-310-darwin.so +0 -0
  84. dbt/compute/smart_selector.py +311 -0
  85. dbt/compute/strategies/__init__.py +54 -0
  86. dbt/compute/strategies/base.py +165 -0
  87. dbt/compute/strategies/dataproc.py +207 -0
  88. dbt/compute/strategies/emr.py +203 -0
  89. dbt/compute/strategies/local.py +364 -0
  90. dbt/compute/strategies/standalone.py +262 -0
  91. dbt/config/__init__.py +4 -0
  92. dbt/config/catalogs.py +94 -0
  93. dbt/config/compute.cpython-310-darwin.so +0 -0
  94. dbt/config/compute.py +547 -0
  95. dbt/config/dvt_profile.cpython-310-darwin.so +0 -0
  96. dbt/config/dvt_profile.py +342 -0
  97. dbt/config/profile.py +422 -0
  98. dbt/config/project.py +873 -0
  99. dbt/config/project_utils.py +28 -0
  100. dbt/config/renderer.py +231 -0
  101. dbt/config/runtime.py +553 -0
  102. dbt/config/selectors.py +208 -0
  103. dbt/config/utils.py +77 -0
  104. dbt/constants.py +28 -0
  105. dbt/context/__init__.py +0 -0
  106. dbt/context/base.py +745 -0
  107. dbt/context/configured.py +135 -0
  108. dbt/context/context_config.py +382 -0
  109. dbt/context/docs.py +82 -0
  110. dbt/context/exceptions_jinja.py +178 -0
  111. dbt/context/macro_resolver.py +195 -0
  112. dbt/context/macros.py +171 -0
  113. dbt/context/manifest.py +72 -0
  114. dbt/context/providers.py +2249 -0
  115. dbt/context/query_header.py +13 -0
  116. dbt/context/secret.py +58 -0
  117. dbt/context/target.py +74 -0
  118. dbt/contracts/__init__.py +0 -0
  119. dbt/contracts/files.py +413 -0
  120. dbt/contracts/graph/__init__.py +0 -0
  121. dbt/contracts/graph/manifest.py +1904 -0
  122. dbt/contracts/graph/metrics.py +97 -0
  123. dbt/contracts/graph/model_config.py +70 -0
  124. dbt/contracts/graph/node_args.py +42 -0
  125. dbt/contracts/graph/nodes.py +1806 -0
  126. dbt/contracts/graph/semantic_manifest.py +232 -0
  127. dbt/contracts/graph/unparsed.py +811 -0
  128. dbt/contracts/project.py +417 -0
  129. dbt/contracts/results.py +53 -0
  130. dbt/contracts/selection.py +23 -0
  131. dbt/contracts/sql.py +85 -0
  132. dbt/contracts/state.py +68 -0
  133. dbt/contracts/util.py +46 -0
  134. dbt/deprecations.py +346 -0
  135. dbt/deps/__init__.py +0 -0
  136. dbt/deps/base.py +152 -0
  137. dbt/deps/git.py +195 -0
  138. dbt/deps/local.py +79 -0
  139. dbt/deps/registry.py +130 -0
  140. dbt/deps/resolver.py +149 -0
  141. dbt/deps/tarball.py +120 -0
  142. dbt/docs/source/_ext/dbt_click.py +119 -0
  143. dbt/docs/source/conf.py +32 -0
  144. dbt/env_vars.py +64 -0
  145. dbt/event_time/event_time.py +40 -0
  146. dbt/event_time/sample_window.py +60 -0
  147. dbt/events/__init__.py +15 -0
  148. dbt/events/base_types.py +36 -0
  149. dbt/events/core_types_pb2.py +2 -0
  150. dbt/events/logging.py +108 -0
  151. dbt/events/types.py +2516 -0
  152. dbt/exceptions.py +1486 -0
  153. dbt/flags.py +89 -0
  154. dbt/graph/__init__.py +11 -0
  155. dbt/graph/cli.py +247 -0
  156. dbt/graph/graph.py +172 -0
  157. dbt/graph/queue.py +214 -0
  158. dbt/graph/selector.py +374 -0
  159. dbt/graph/selector_methods.py +975 -0
  160. dbt/graph/selector_spec.py +222 -0
  161. dbt/graph/thread_pool.py +18 -0
  162. dbt/hooks.py +21 -0
  163. dbt/include/README.md +49 -0
  164. dbt/include/__init__.py +3 -0
  165. dbt/include/starter_project/.gitignore +4 -0
  166. dbt/include/starter_project/README.md +15 -0
  167. dbt/include/starter_project/__init__.py +3 -0
  168. dbt/include/starter_project/analyses/.gitkeep +0 -0
  169. dbt/include/starter_project/dbt_project.yml +36 -0
  170. dbt/include/starter_project/macros/.gitkeep +0 -0
  171. dbt/include/starter_project/models/example/my_first_dbt_model.sql +27 -0
  172. dbt/include/starter_project/models/example/my_second_dbt_model.sql +6 -0
  173. dbt/include/starter_project/models/example/schema.yml +21 -0
  174. dbt/include/starter_project/seeds/.gitkeep +0 -0
  175. dbt/include/starter_project/snapshots/.gitkeep +0 -0
  176. dbt/include/starter_project/tests/.gitkeep +0 -0
  177. dbt/internal_deprecations.py +26 -0
  178. dbt/jsonschemas/__init__.py +3 -0
  179. dbt/jsonschemas/jsonschemas.py +309 -0
  180. dbt/jsonschemas/project/0.0.110.json +4717 -0
  181. dbt/jsonschemas/project/0.0.85.json +2015 -0
  182. dbt/jsonschemas/resources/0.0.110.json +2636 -0
  183. dbt/jsonschemas/resources/0.0.85.json +2536 -0
  184. dbt/jsonschemas/resources/latest.json +6773 -0
  185. dbt/links.py +4 -0
  186. dbt/materializations/__init__.py +0 -0
  187. dbt/materializations/incremental/__init__.py +0 -0
  188. dbt/materializations/incremental/microbatch.py +236 -0
  189. dbt/mp_context.py +8 -0
  190. dbt/node_types.py +37 -0
  191. dbt/parser/__init__.py +23 -0
  192. dbt/parser/analysis.py +21 -0
  193. dbt/parser/base.py +548 -0
  194. dbt/parser/common.py +266 -0
  195. dbt/parser/docs.py +52 -0
  196. dbt/parser/fixtures.py +51 -0
  197. dbt/parser/functions.py +30 -0
  198. dbt/parser/generic_test.py +100 -0
  199. dbt/parser/generic_test_builders.py +333 -0
  200. dbt/parser/hooks.py +118 -0
  201. dbt/parser/macros.py +137 -0
  202. dbt/parser/manifest.py +2204 -0
  203. dbt/parser/models.py +573 -0
  204. dbt/parser/partial.py +1178 -0
  205. dbt/parser/read_files.py +445 -0
  206. dbt/parser/schema_generic_tests.py +422 -0
  207. dbt/parser/schema_renderer.py +111 -0
  208. dbt/parser/schema_yaml_readers.py +935 -0
  209. dbt/parser/schemas.py +1466 -0
  210. dbt/parser/search.py +149 -0
  211. dbt/parser/seeds.py +28 -0
  212. dbt/parser/singular_test.py +20 -0
  213. dbt/parser/snapshots.py +44 -0
  214. dbt/parser/sources.py +558 -0
  215. dbt/parser/sql.py +62 -0
  216. dbt/parser/unit_tests.py +621 -0
  217. dbt/plugins/__init__.py +20 -0
  218. dbt/plugins/contracts.py +9 -0
  219. dbt/plugins/exceptions.py +2 -0
  220. dbt/plugins/manager.py +163 -0
  221. dbt/plugins/manifest.py +21 -0
  222. dbt/profiler.py +20 -0
  223. dbt/py.typed +1 -0
  224. dbt/query_analyzer.cpython-310-darwin.so +0 -0
  225. dbt/query_analyzer.py +410 -0
  226. dbt/runners/__init__.py +2 -0
  227. dbt/runners/exposure_runner.py +7 -0
  228. dbt/runners/no_op_runner.py +45 -0
  229. dbt/runners/saved_query_runner.py +7 -0
  230. dbt/selected_resources.py +8 -0
  231. dbt/task/__init__.py +0 -0
  232. dbt/task/base.py +503 -0
  233. dbt/task/build.py +197 -0
  234. dbt/task/clean.py +56 -0
  235. dbt/task/clone.py +161 -0
  236. dbt/task/compile.py +150 -0
  237. dbt/task/compute.py +454 -0
  238. dbt/task/debug.py +505 -0
  239. dbt/task/deps.py +280 -0
  240. dbt/task/docs/__init__.py +3 -0
  241. dbt/task/docs/generate.py +660 -0
  242. dbt/task/docs/index.html +250 -0
  243. dbt/task/docs/serve.py +29 -0
  244. dbt/task/freshness.py +322 -0
  245. dbt/task/function.py +121 -0
  246. dbt/task/group_lookup.py +46 -0
  247. dbt/task/init.py +553 -0
  248. dbt/task/java.py +316 -0
  249. dbt/task/list.py +236 -0
  250. dbt/task/printer.py +175 -0
  251. dbt/task/retry.py +175 -0
  252. dbt/task/run.py +1306 -0
  253. dbt/task/run_operation.py +141 -0
  254. dbt/task/runnable.py +758 -0
  255. dbt/task/seed.py +103 -0
  256. dbt/task/show.py +149 -0
  257. dbt/task/snapshot.py +56 -0
  258. dbt/task/spark.py +414 -0
  259. dbt/task/sql.py +110 -0
  260. dbt/task/target_sync.py +759 -0
  261. dbt/task/test.py +464 -0
  262. dbt/tests/fixtures/__init__.py +1 -0
  263. dbt/tests/fixtures/project.py +620 -0
  264. dbt/tests/util.py +651 -0
  265. dbt/tracking.py +529 -0
  266. dbt/utils/__init__.py +3 -0
  267. dbt/utils/artifact_upload.py +151 -0
  268. dbt/utils/utils.py +408 -0
  269. dbt/version.py +268 -0
  270. dvt_cli/__init__.py +72 -0
  271. dvt_core-0.52.2.dist-info/METADATA +286 -0
  272. dvt_core-0.52.2.dist-info/RECORD +275 -0
  273. dvt_core-0.52.2.dist-info/WHEEL +5 -0
  274. dvt_core-0.52.2.dist-info/entry_points.txt +2 -0
  275. dvt_core-0.52.2.dist-info/top_level.txt +2 -0
dbt/parser/schemas.py ADDED
@@ -0,0 +1,1466 @@
1
+ import datetime
2
+ import pathlib
3
+ import re
4
+ import time
5
+ from abc import ABCMeta, abstractmethod
6
+ from dataclasses import dataclass, field
7
+ from typing import (
8
+ Any,
9
+ Callable,
10
+ Dict,
11
+ Generic,
12
+ Iterable,
13
+ List,
14
+ Optional,
15
+ Tuple,
16
+ Type,
17
+ TypeVar,
18
+ )
19
+
20
+ from dbt.artifacts.resources import CustomGranularity, RefArgs, TimeSpine
21
+ from dbt.clients.checked_load import (
22
+ checked_load,
23
+ issue_deprecation_warnings_for_failures,
24
+ )
25
+ from dbt.clients.jinja_static import statically_parse_ref_or_source
26
+ from dbt.clients.yaml_helper import load_yaml_text
27
+ from dbt.config import RuntimeConfig
28
+ from dbt.context.configured import SchemaYamlVars, generate_schema_yml_context
29
+ from dbt.context.context_config import ContextConfig
30
+ from dbt.contracts.files import SchemaSourceFile, SourceFile
31
+ from dbt.contracts.graph.manifest import Manifest
32
+ from dbt.contracts.graph.nodes import (
33
+ FunctionNode,
34
+ Macro,
35
+ ModelNode,
36
+ ParsedFunctionPatch,
37
+ ParsedMacroPatch,
38
+ ParsedNodePatch,
39
+ ParsedSingularTestPatch,
40
+ UnpatchedSourceDefinition,
41
+ )
42
+ from dbt.contracts.graph.unparsed import (
43
+ HasColumnDocs,
44
+ HasColumnTests,
45
+ SourcePatch,
46
+ UnparsedAnalysisUpdate,
47
+ UnparsedFunctionUpdate,
48
+ UnparsedMacroUpdate,
49
+ UnparsedModelUpdate,
50
+ UnparsedNodeUpdate,
51
+ UnparsedSingularTestUpdate,
52
+ UnparsedSourceDefinition,
53
+ )
54
+ from dbt.events.types import (
55
+ InvalidMacroAnnotation,
56
+ MacroNotFoundForPatch,
57
+ NoNodeForYamlKey,
58
+ UnsupportedConstraintMaterialization,
59
+ ValidationWarning,
60
+ WrongResourceSchemaFile,
61
+ )
62
+ from dbt.exceptions import (
63
+ DbtInternalError,
64
+ DuplicateMacroPatchNameError,
65
+ DuplicatePatchPathError,
66
+ DuplicateSourcePatchNameError,
67
+ InvalidAccessTypeError,
68
+ JSONValidationError,
69
+ ParsingError,
70
+ YamlLoadError,
71
+ YamlParseDictError,
72
+ YamlParseListError,
73
+ )
74
+ from dbt.flags import get_flags
75
+ from dbt.node_types import AccessType, NodeType
76
+ from dbt.parser.base import SimpleParser
77
+ from dbt.parser.common import (
78
+ ParserRef,
79
+ TargetBlock,
80
+ TestBlock,
81
+ VersionedTestBlock,
82
+ YamlBlock,
83
+ schema_file_keys_to_resource_types,
84
+ trimmed,
85
+ )
86
+ from dbt.parser.schema_generic_tests import SchemaGenericTestParser
87
+ from dbt.parser.schema_renderer import SchemaYamlRenderer
88
+ from dbt.parser.search import FileBlock
89
+ from dbt.utils import coerce_dict_str
90
+ from dbt_common.contracts.constraints import ConstraintType, ModelLevelConstraint
91
+ from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
92
+ from dbt_common.events import EventLevel
93
+ from dbt_common.events.functions import fire_event, warn_or_error
94
+ from dbt_common.events.types import Note
95
+ from dbt_common.exceptions import DbtValidationError
96
+ from dbt_common.utils import deep_merge
97
+
98
+ # ===============================================================================
99
+ # Schema Parser classes
100
+ #
101
+ # The SchemaParser is a subclass of the SimpleParser from base.py, as is
102
+ # the SchemaGenericTestParser. The schema sub-parsers are all subclasses of
103
+ # the YamlReader parsing class. Most of the action in creating SourceDefinition
104
+ # nodes actually happens in the SourcePatcher class, in sources.py, which is
105
+ # called as a late-stage parsing step in manifest.py.
106
+ #
107
+ # The "patch" parsers read yaml config and properties and apply them to
108
+ # nodes that were already created from sql files.
109
+ #
110
+ # The SchemaParser and SourcePatcher both use the SchemaGenericTestParser
111
+ # (in schema_generic_tests.py) to create generic test nodes.
112
+ #
113
+ # YamlReader
114
+ # MetricParser (metrics) [schema_yaml_readers.py]
115
+ # ExposureParser (exposures) [schema_yaml_readers.py]
116
+ # GroupParser (groups) [schema_yaml_readers.py]
117
+ # SourceParser (sources)
118
+ # PatchParser
119
+ # MacroPatchParser (macros)
120
+ # NodePatchParser
121
+ # ModelPatchParser (models)
122
+ # AnalysisPatchParser (analyses)
123
+ # TestablePatchParser (seeds, snapshots)
124
+ #
125
+ # ===============================================================================
126
+
127
+
128
+ def yaml_from_file(
129
+ source_file: SchemaSourceFile, validate: bool = False
130
+ ) -> Optional[Dict[str, Any]]:
131
+ """If loading the yaml fails, raise an exception."""
132
+ try:
133
+ # source_file.contents can sometimes be None
134
+ to_load = source_file.contents or ""
135
+
136
+ if validate:
137
+ contents, failures = checked_load(to_load)
138
+ issue_deprecation_warnings_for_failures(
139
+ failures=failures, file=source_file.path.original_file_path
140
+ )
141
+ if contents is not None:
142
+ from dbt.jsonschemas.jsonschemas import (
143
+ jsonschema_validate,
144
+ resources_schema,
145
+ )
146
+
147
+ # Validate the yaml against the jsonschema to raise deprecation warnings
148
+ # for invalid fields.
149
+ jsonschema_validate(
150
+ schema=resources_schema(),
151
+ json=contents,
152
+ file_path=source_file.path.original_file_path,
153
+ )
154
+ else:
155
+ contents = load_yaml_text(to_load, source_file.path)
156
+
157
+ if contents is None:
158
+ return contents
159
+
160
+ if not isinstance(contents, dict):
161
+ raise DbtValidationError(
162
+ f"Contents of file '{source_file.original_file_path}' are not valid. Dictionary expected."
163
+ )
164
+
165
+ # When loaded_at_field is defined as None or null, it shows up in
166
+ # the dict but when it is not defined, it does not show up in the dict
167
+ # We need to capture this to be able to override source level settings later.
168
+ for source in contents.get("sources", []):
169
+ for table in source.get("tables", []):
170
+ if "loaded_at_field" in table or (
171
+ "config" in table
172
+ and table["config"] is not None
173
+ and table["config"].get("loaded_at_field")
174
+ ):
175
+ table["loaded_at_field_present"] = True
176
+
177
+ return contents
178
+ except DbtValidationError as e:
179
+ raise YamlLoadError(
180
+ project_name=source_file.project_name, path=source_file.path.relative_path, exc=e
181
+ )
182
+
183
+
184
+ # This is the main schema file parser, but almost everything happens in the
185
+ # the schema sub-parsers.
186
+ class SchemaParser(SimpleParser[YamlBlock, ModelNode]):
187
+ def __init__(
188
+ self,
189
+ project: RuntimeConfig,
190
+ manifest: Manifest,
191
+ root_project: RuntimeConfig,
192
+ ) -> None:
193
+ super().__init__(project, manifest, root_project)
194
+
195
+ self.generic_test_parser = SchemaGenericTestParser(project, manifest, root_project)
196
+
197
+ self.schema_yaml_vars = SchemaYamlVars()
198
+ self.render_ctx = generate_schema_yml_context(
199
+ self.root_project, self.project.project_name, self.schema_yaml_vars
200
+ )
201
+
202
+ # This is unnecessary, but mypy was requiring it. Clean up parser code so
203
+ # we don't have to do this.
204
+ def parse_from_dict(self, dct):
205
+ pass
206
+
207
+ @classmethod
208
+ def get_compiled_path(cls, block: FileBlock) -> str:
209
+ # should this raise an error?
210
+ return block.path.relative_path
211
+
212
+ @property
213
+ def resource_type(self) -> NodeType:
214
+ return NodeType.Test
215
+
216
+ def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None:
217
+ assert isinstance(block.file, SchemaSourceFile)
218
+
219
+ # If partially parsing, dct should be from pp_dict, otherwise
220
+ # dict_from_yaml
221
+ if dct:
222
+ # contains the FileBlock and the data (dictionary)
223
+ yaml_block = YamlBlock.from_file_block(block, dct)
224
+ parser: YamlReader
225
+
226
+ # There are 9 different yaml lists which are parsed by different parsers:
227
+ # Model, Seed, Snapshot, Source, Macro, Analysis, Exposure, Metric, Group
228
+
229
+ # ModelPatchParser.parse()
230
+ if "models" in dct:
231
+ # the models are already in the manifest as nodes when we reach this code,
232
+ # even if they are disabled in the schema file
233
+ model_parse_result = ModelPatchParser(self, yaml_block, "models").parse()
234
+ for versioned_test_block in model_parse_result.versioned_test_blocks:
235
+ self.generic_test_parser.parse_versioned_tests(versioned_test_block)
236
+
237
+ # PatchParser.parse()
238
+ if "seeds" in dct:
239
+ seed_parse_result = TestablePatchParser(self, yaml_block, "seeds").parse()
240
+ for test_block in seed_parse_result.test_blocks:
241
+ self.generic_test_parser.parse_tests(test_block)
242
+
243
+ # PatchParser.parse()
244
+ if "snapshots" in dct:
245
+ self._add_yaml_snapshot_nodes_to_manifest(dct["snapshots"], block)
246
+ snapshot_parse_result = TestablePatchParser(self, yaml_block, "snapshots").parse()
247
+ for test_block in snapshot_parse_result.test_blocks:
248
+ self.generic_test_parser.parse_tests(test_block)
249
+
250
+ # This parser uses SourceParser.parse() which doesn't return
251
+ # any test blocks. Source tests are handled at a later point
252
+ # in the process.
253
+ if "sources" in dct:
254
+ parser = SourceParser(self, yaml_block, "sources")
255
+ parser.parse()
256
+
257
+ # PatchParser.parse() (but never test_blocks)
258
+ if "macros" in dct:
259
+ parser = MacroPatchParser(self, yaml_block, "macros")
260
+ parser.parse()
261
+
262
+ if "data_tests" in dct:
263
+ parser = SingularTestPatchParser(self, yaml_block, "data_tests")
264
+ try:
265
+ parser.parse()
266
+ except ParsingError as e:
267
+ fire_event(
268
+ Note(
269
+ msg=f"Unable to parse 'data_tests' section of file '{block.path.original_file_path}'\n{e}",
270
+ ),
271
+ EventLevel.WARN,
272
+ )
273
+
274
+ # PatchParser.parse() (but never test_blocks)
275
+ if "analyses" in dct:
276
+ parser = AnalysisPatchParser(self, yaml_block, "analyses")
277
+ parser.parse()
278
+
279
+ # ExposureParser.parse()
280
+ if "exposures" in dct:
281
+ from dbt.parser.schema_yaml_readers import ExposureParser
282
+
283
+ exp_parser = ExposureParser(self, yaml_block)
284
+ exp_parser.parse()
285
+
286
+ # FunctionPatchParser.parse()
287
+ if "functions" in dct:
288
+ function_parser = FunctionPatchParser(self, yaml_block, "functions")
289
+ function_parser.parse()
290
+
291
+ # MetricParser.parse()
292
+ if "metrics" in dct:
293
+ from dbt.parser.schema_yaml_readers import MetricParser
294
+
295
+ metric_parser = MetricParser(self, yaml_block)
296
+ metric_parser.parse()
297
+
298
+ # GroupParser.parse()
299
+ if "groups" in dct:
300
+ from dbt.parser.schema_yaml_readers import GroupParser
301
+
302
+ group_parser = GroupParser(self, yaml_block)
303
+ group_parser.parse()
304
+
305
+ if "semantic_models" in dct:
306
+ from dbt.parser.schema_yaml_readers import SemanticModelParser
307
+
308
+ semantic_model_parser = SemanticModelParser(self, yaml_block)
309
+ semantic_model_parser.parse()
310
+
311
+ if "unit_tests" in dct:
312
+ from dbt.parser.unit_tests import UnitTestParser
313
+
314
+ unit_test_parser = UnitTestParser(self, yaml_block)
315
+ unit_test_parser.parse()
316
+
317
+ if "saved_queries" in dct:
318
+ from dbt.parser.schema_yaml_readers import SavedQueryParser
319
+
320
+ saved_query_parser = SavedQueryParser(self, yaml_block)
321
+ saved_query_parser.parse()
322
+
323
+ def _add_yaml_snapshot_nodes_to_manifest(
324
+ self, snapshots: List[Dict[str, Any]], block: FileBlock
325
+ ) -> None:
326
+ """We support the creation of simple snapshots in yaml, without an
327
+ accompanying SQL definition. For such snapshots, the user must supply
328
+ a 'relation' property to indicate the target of the snapshot. This
329
+ function looks for such snapshots and adds a node to manifest for each
330
+ one we find, since they were not added during SQL parsing."""
331
+
332
+ rebuild_refs = False
333
+ for snapshot in snapshots:
334
+ if "relation" in snapshot:
335
+ from dbt.parser import SnapshotParser
336
+
337
+ if "name" not in snapshot:
338
+ raise ParsingError("A snapshot must define the 'name' property. ")
339
+
340
+ # Reuse the logic of SnapshotParser as far as possible to create
341
+ # a new node we can add to the manifest.
342
+ parser = SnapshotParser(self.project, self.manifest, self.root_project)
343
+ fqn = parser.get_fqn_prefix(block.path.relative_path)
344
+ fqn.append(snapshot["name"])
345
+
346
+ compiled_path = str(
347
+ pathlib.PurePath("").joinpath(
348
+ block.path.relative_path, snapshot["name"] + ".sql"
349
+ )
350
+ )
351
+ snapshot_node = parser._create_parsetime_node(
352
+ block,
353
+ compiled_path,
354
+ parser.initial_config(fqn),
355
+ fqn,
356
+ snapshot["name"],
357
+ )
358
+
359
+ # Parse the expected ref() or source() expression given by
360
+ # 'relation' so that we know what we are snapshotting.
361
+ source_or_ref = statically_parse_ref_or_source(snapshot["relation"])
362
+ if isinstance(source_or_ref, RefArgs):
363
+ snapshot_node.refs.append(source_or_ref)
364
+ else:
365
+ snapshot_node.sources.append(source_or_ref)
366
+
367
+ # Implement the snapshot SQL as a simple select *
368
+ snapshot_node.raw_code = "select * from {{ " + snapshot["relation"] + " }}"
369
+
370
+ # Add our new node to the manifest, and note that ref lookup collections
371
+ # will need to be rebuilt. This adds the node unique_id to the "snapshots"
372
+ # list in the SchemaSourceFile.
373
+ self.manifest.add_node(block.file, snapshot_node)
374
+ rebuild_refs = True
375
+
376
+ if rebuild_refs:
377
+ self.manifest.rebuild_ref_lookup()
378
+
379
+
380
+ Parsed = TypeVar(
381
+ "Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch, ParsedSingularTestPatch
382
+ )
383
+ NodeTarget = TypeVar(
384
+ "NodeTarget",
385
+ UnparsedNodeUpdate,
386
+ UnparsedAnalysisUpdate,
387
+ UnparsedModelUpdate,
388
+ UnparsedFunctionUpdate,
389
+ )
390
+ NonSourceTarget = TypeVar(
391
+ "NonSourceTarget",
392
+ UnparsedNodeUpdate,
393
+ UnparsedAnalysisUpdate,
394
+ UnparsedMacroUpdate,
395
+ UnparsedModelUpdate,
396
+ UnparsedFunctionUpdate,
397
+ UnparsedSingularTestUpdate,
398
+ )
399
+
400
+
401
+ @dataclass
402
+ class ParseResult:
403
+ test_blocks: List[TestBlock] = field(default_factory=list)
404
+ versioned_test_blocks: List[VersionedTestBlock] = field(default_factory=list)
405
+
406
+
407
+ # abstract base class (ABCMeta)
408
+ # Many subclasses: MetricParser, ExposureParser, GroupParser, SourceParser,
409
+ # PatchParser, SemanticModelParser, SavedQueryParser, UnitTestParser
410
+ class YamlReader(metaclass=ABCMeta):
411
+ def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock, key: str) -> None:
412
+ self.schema_parser: SchemaParser = schema_parser
413
+ # key: models, seeds, snapshots, sources, macros,
414
+ # analyses, exposures, unit_tests
415
+ self.key: str = key
416
+ self.yaml: YamlBlock = yaml
417
+ self.schema_yaml_vars: SchemaYamlVars = SchemaYamlVars()
418
+ self.render_ctx = generate_schema_yml_context(
419
+ self.schema_parser.root_project,
420
+ self.schema_parser.project.project_name,
421
+ self.schema_yaml_vars,
422
+ )
423
+ self.renderer: SchemaYamlRenderer = SchemaYamlRenderer(self.render_ctx, self.key)
424
+
425
+ @property
426
+ def manifest(self) -> Manifest:
427
+ return self.schema_parser.manifest
428
+
429
+ @property
430
+ def project(self) -> RuntimeConfig:
431
+ return self.schema_parser.project
432
+
433
+ @property
434
+ def default_database(self) -> str:
435
+ return self.schema_parser.default_database
436
+
437
+ @property
438
+ def root_project(self) -> RuntimeConfig:
439
+ return self.schema_parser.root_project
440
+
441
+ # for the different schema subparsers ('models', 'source', etc)
442
+ # get the list of dicts pointed to by the key in the yaml config,
443
+ # ensure that the dicts have string keys
444
+ def get_key_dicts(self) -> Iterable[Dict[str, Any]]:
445
+ data = self.yaml.data.get(self.key, [])
446
+ if not isinstance(data, list):
447
+ raise ParsingError(
448
+ "{} must be a list, got {} instead: ({})".format(
449
+ self.key, type(data), trimmed(str(data))
450
+ )
451
+ )
452
+ path = self.yaml.path.original_file_path
453
+
454
+ # for each dict in the data (which is a list of dicts)
455
+ for entry in data:
456
+
457
+ # check that entry is a dict and that all dict values
458
+ # are strings
459
+ if coerce_dict_str(entry) is None:
460
+ raise YamlParseListError(path, self.key, data, "expected a dict with string keys")
461
+
462
+ if "name" not in entry and "model" not in entry:
463
+ raise ParsingError("Entry did not contain a name")
464
+
465
+ unrendered_config = {}
466
+ if "config" in entry:
467
+ unrendered_config = entry["config"]
468
+
469
+ unrendered_version_configs = {}
470
+ if "versions" in entry:
471
+ for version in entry["versions"]:
472
+ if "v" in version:
473
+ unrendered_version_configs[version["v"]] = version.get("config", {})
474
+
475
+ # For sources
476
+ unrendered_database = entry.get("database", None)
477
+ unrendered_schema = entry.get("schema", None)
478
+
479
+ # Render the data (except for tests, data_tests and descriptions).
480
+ # See the SchemaYamlRenderer
481
+ entry = self.render_entry(entry)
482
+
483
+ schema_file = self.yaml.file
484
+ assert isinstance(schema_file, SchemaSourceFile)
485
+
486
+ if unrendered_config:
487
+ schema_file.add_unrendered_config(unrendered_config, self.key, entry["name"])
488
+
489
+ for version, unrendered_version_config in unrendered_version_configs.items():
490
+ schema_file.add_unrendered_config(
491
+ unrendered_version_config, self.key, entry["name"], version
492
+ )
493
+
494
+ if unrendered_database:
495
+ schema_file.add_unrendered_database(self.key, entry["name"], unrendered_database)
496
+ if unrendered_schema:
497
+ schema_file.add_unrendered_schema(self.key, entry["name"], unrendered_schema)
498
+
499
+ if self.schema_yaml_vars.env_vars:
500
+ self.schema_parser.manifest.env_vars.update(self.schema_yaml_vars.env_vars)
501
+ for var in self.schema_yaml_vars.env_vars.keys():
502
+ schema_file.add_env_var(var, self.key, entry["name"])
503
+ self.schema_yaml_vars.env_vars = {}
504
+
505
+ yield entry
506
+
507
+ def render_entry(self, dct):
508
+ try:
509
+ # This does a deep_map which will fail if there are circular references
510
+ dct = self.renderer.render_data(dct)
511
+ except ParsingError as exc:
512
+ raise ParsingError(
513
+ f"Failed to render {self.yaml.file.path.original_file_path} from "
514
+ f"project {self.project.project_name}: {exc}"
515
+ ) from exc
516
+ return dct
517
+
518
+ @abstractmethod
519
+ def parse(self) -> Optional[ParseResult]:
520
+ raise NotImplementedError("parse is abstract")
521
+
522
+
523
+ T = TypeVar("T", bound=dbtClassMixin)
524
+
525
+
526
+ # This parses the 'sources' keys in yaml files.
527
+ class SourceParser(YamlReader):
528
+ def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T:
529
+ path = self.yaml.path.original_file_path
530
+ try:
531
+ cls.validate(data)
532
+ return cls.from_dict(data)
533
+ except (ValidationError, JSONValidationError) as exc:
534
+ raise YamlParseDictError(path, self.key, data, exc)
535
+
536
+ # This parse method takes the yaml dictionaries in 'sources' keys and uses them
537
+ # to create UnparsedSourceDefinition objects. They are then turned
538
+ # into UnpatchedSourceDefinition objects in 'add_source_definitions'
539
+ # or SourcePatch objects in 'add_source_patch'
540
+ def parse(self) -> ParseResult:
541
+ # get a verified list of dicts for the key handled by this parser
542
+ for data in self.get_key_dicts():
543
+ data = self.project.credentials.translate_aliases(data, recurse=True)
544
+
545
+ is_override = "overrides" in data
546
+ if is_override:
547
+ data["path"] = self.yaml.path.original_file_path
548
+ patch = self._target_from_dict(SourcePatch, data)
549
+ assert isinstance(self.yaml.file, SchemaSourceFile)
550
+ source_file = self.yaml.file
551
+ # source patches must be unique
552
+ key = (patch.overrides, patch.name)
553
+ if key in self.manifest.source_patches:
554
+ raise DuplicateSourcePatchNameError(patch, self.manifest.source_patches[key])
555
+ self.manifest.source_patches[key] = patch
556
+ source_file.source_patches.append(key)
557
+ else:
558
+ source = self._target_from_dict(UnparsedSourceDefinition, data)
559
+ # Store unrendered_database and unrendered_schema for state:modified comparisons
560
+ if isinstance(self.yaml.file, SchemaSourceFile):
561
+ source.unrendered_database = self.yaml.file.get_unrendered_database(
562
+ "sources", source.name
563
+ )
564
+ source.unrendered_schema = self.yaml.file.get_unrendered_schema(
565
+ "sources", source.name
566
+ )
567
+
568
+ self.add_source_definitions(source)
569
+ return ParseResult()
570
+
571
+ def add_source_definitions(self, source: UnparsedSourceDefinition) -> None:
572
+ package_name = self.project.project_name
573
+ original_file_path = self.yaml.path.original_file_path
574
+ fqn_path = self.yaml.path.relative_path
575
+ for table in source.tables:
576
+ unique_id = ".".join([NodeType.Source, package_name, source.name, table.name])
577
+
578
+ # the FQN is project name / path elements /source_name /table_name
579
+ fqn = self.schema_parser.get_fqn_prefix(fqn_path)
580
+ fqn.extend([source.name, table.name])
581
+
582
+ source_def = UnpatchedSourceDefinition(
583
+ source=source,
584
+ table=table,
585
+ path=original_file_path,
586
+ original_file_path=original_file_path,
587
+ package_name=package_name,
588
+ unique_id=unique_id,
589
+ resource_type=NodeType.Source,
590
+ fqn=fqn,
591
+ name=f"{source.name}_{table.name}",
592
+ )
593
+ assert isinstance(self.yaml.file, SchemaSourceFile)
594
+ source_file: SchemaSourceFile = self.yaml.file
595
+ self.manifest.add_source(source_file, source_def)
596
+
597
+
598
+ # This class has two subclasses: NodePatchParser and MacroPatchParser
599
+ class PatchParser(YamlReader, Generic[NonSourceTarget, Parsed]):
600
+ @abstractmethod
601
+ def _target_type(self) -> Type[NonSourceTarget]:
602
+ raise NotImplementedError("_target_type not implemented")
603
+
604
+ @abstractmethod
605
+ def get_block(self, node: NonSourceTarget) -> TargetBlock:
606
+ raise NotImplementedError("get_block is abstract")
607
+
608
+ @abstractmethod
609
+ def parse_patch(self, block: TargetBlock[NonSourceTarget], refs: ParserRef) -> None:
610
+ raise NotImplementedError("parse_patch is abstract")
611
+
612
+ def parse(self) -> ParseResult:
613
+ node: NonSourceTarget
614
+ # This will always be empty if the node a macro or analysis
615
+ test_blocks: List[TestBlock] = []
616
+ # This will always be empty if the node is _not_ a model
617
+ versioned_test_blocks: List[VersionedTestBlock] = []
618
+
619
+ # get list of 'node' objects
620
+ # UnparsedNodeUpdate (TestablePatchParser, models, seeds, snapshots)
621
+ # = HasColumnTests, HasTests
622
+ # UnparsedAnalysisUpdate (UnparsedAnalysisParser, analyses)
623
+ # = HasColumnDocs, HasDocs
624
+ # UnparsedMacroUpdate (MacroPatchParser, 'macros')
625
+ # = HasDocs
626
+ # correspond to this parser's 'key'
627
+ for node in self.get_unparsed_target():
628
+ # node_block is a TargetBlock (Macro or Analysis)
629
+ # or a TestBlock (all of the others)
630
+ node_block = self.get_block(node)
631
+ if isinstance(node_block, TestBlock):
632
+ # TestablePatchParser = seeds, snapshots
633
+ test_blocks.append(node_block)
634
+ if isinstance(node_block, VersionedTestBlock):
635
+ # models
636
+ versioned_test_blocks.append(node_block)
637
+ if isinstance(node, (HasColumnDocs, HasColumnTests)):
638
+ # UnparsedNodeUpdate and UnparsedAnalysisUpdate
639
+ refs: ParserRef = ParserRef.from_target(node)
640
+ else:
641
+ refs = ParserRef()
642
+
643
+ # There's no unique_id on the node yet so cannot add to disabled dict
644
+ self.parse_patch(node_block, refs)
645
+
646
+ return ParseResult(test_blocks, versioned_test_blocks)
647
+
648
+ def get_unparsed_target(self) -> Iterable[NonSourceTarget]:
649
+ path = self.yaml.path.original_file_path
650
+
651
+ # get verified list of dicts for the 'key' that this
652
+ # parser handles
653
+ key_dicts = self.get_key_dicts()
654
+ for data in key_dicts:
655
+ # add extra data to each dict. This updates the dicts
656
+ # in the parser yaml
657
+ data.update(
658
+ {
659
+ "original_file_path": path,
660
+ "yaml_key": self.key,
661
+ "package_name": self.project.project_name,
662
+ }
663
+ )
664
+ try:
665
+ # target_type: UnparsedNodeUpdate, UnparsedAnalysisUpdate,
666
+ # or UnparsedMacroUpdate, UnparsedFunctionUpdate
667
+ self._target_type().validate(data)
668
+ if self.key != "macros":
669
+ # macros don't have the 'config' key support yet
670
+ self.normalize_meta_attribute(data, path)
671
+ self.normalize_docs_attribute(data, path)
672
+ self.normalize_group_attribute(data, path)
673
+ self.normalize_contract_attribute(data, path)
674
+ self.normalize_access_attribute(data, path)
675
+ # `tests` has been deprecated, convert to `data_tests` here if present
676
+ self.validate_data_tests(data)
677
+ node = self._target_type().from_dict(data)
678
+ except (ValidationError, JSONValidationError) as exc:
679
+ raise YamlParseDictError(path, self.key, data, exc)
680
+ else:
681
+ yield node
682
+
683
+ # We want to raise an error if some attributes are in two places, and move them
684
+ # from toplevel to config if necessary
685
+ def normalize_attribute(self, data, path, attribute) -> None:
686
+ if attribute in data:
687
+ if "config" in data and attribute in data["config"]:
688
+ raise ParsingError(
689
+ f"""
690
+ In {path}: found {attribute} dictionary in 'config' dictionary and as top-level key.
691
+ Remove the top-level key and define it under 'config' dictionary only.
692
+ """.strip()
693
+ )
694
+ else:
695
+ if "config" not in data:
696
+ data["config"] = {}
697
+ data["config"][attribute] = data.pop(attribute)
698
+
699
+ def normalize_meta_attribute(self, data, path) -> None:
700
+ return self.normalize_attribute(data, path, "meta")
701
+
702
+ def normalize_docs_attribute(self, data, path) -> None:
703
+ return self.normalize_attribute(data, path, "docs")
704
+
705
+ def normalize_group_attribute(self, data, path) -> None:
706
+ return self.normalize_attribute(data, path, "group")
707
+
708
+ def normalize_contract_attribute(self, data, path) -> None:
709
+ return self.normalize_attribute(data, path, "contract")
710
+
711
+ def normalize_access_attribute(self, data, path) -> None:
712
+ return self.normalize_attribute(data, path, "access")
713
+
714
+ @property
715
+ def is_root_project(self) -> bool:
716
+ if self.root_project.project_name == self.project.project_name:
717
+ return True
718
+ return False
719
+
720
+ def validate_data_tests(self, data) -> None:
721
+ # Rename 'tests' -> 'data_tests' at both model-level and column-level
722
+ # Raise a validation error if the user has defined both names
723
+ def validate_and_rename(data, is_root_project: bool) -> None:
724
+ if data.get("tests"):
725
+ if "tests" in data and "data_tests" in data:
726
+ raise ValidationError(
727
+ "Invalid test config: cannot have both 'tests' and 'data_tests' defined"
728
+ )
729
+ data["data_tests"] = data.pop("tests")
730
+
731
+ # model-level tests
732
+ validate_and_rename(data, self.is_root_project)
733
+
734
+ # column-level tests
735
+ if data.get("columns"):
736
+ for column in data["columns"]:
737
+ validate_and_rename(column, self.is_root_project)
738
+
739
+ # versioned models
740
+ if data.get("versions"):
741
+ for version in data["versions"]:
742
+ validate_and_rename(version, self.is_root_project)
743
+ if version.get("columns"):
744
+ for column in version["columns"]:
745
+ validate_and_rename(column, self.is_root_project)
746
+
747
+ def patch_node_config(self, node, patch) -> None:
748
+ if "access" in patch.config:
749
+ if AccessType.is_valid(patch.config["access"]):
750
+ patch.config["access"] = AccessType(patch.config["access"])
751
+ else:
752
+ raise InvalidAccessTypeError(
753
+ unique_id=node.unique_id,
754
+ field_value=patch.config["access"],
755
+ )
756
+ # Get the ContextConfig that's used in calculating the config
757
+ # This must match the model resource_type that's being patched
758
+ config = ContextConfig(
759
+ self.schema_parser.root_project,
760
+ node.fqn,
761
+ node.resource_type,
762
+ self.schema_parser.project.project_name,
763
+ )
764
+ # We need to re-apply the config_call_dict after the patch config
765
+ config._config_call_dict = node.config_call_dict
766
+ config._unrendered_config_call_dict = node.unrendered_config_call_dict
767
+ self.schema_parser.update_parsed_node_config(
768
+ node,
769
+ config,
770
+ patch_config_dict=patch.config,
771
+ patch_file_id=patch.file_id,
772
+ )
773
+
774
+
775
+ # Subclasses of NodePatchParser: TestablePatchParser, ModelPatchParser, AnalysisPatchParser, FunctionPatchParser
776
+ # so models, seeds, snapshots, analyses, functions
777
+ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarget]):
778
+ def _get_node_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> ParsedNodePatch:
779
+ # We're not passing the ParsedNodePatch around anymore, so we
780
+ # could possibly skip creating one. Leaving here for now for
781
+ # code consistency.
782
+ deprecation_date: Optional[datetime.datetime] = None
783
+ time_spine: Optional[TimeSpine] = None
784
+
785
+ if isinstance(block.target, UnparsedModelUpdate):
786
+ deprecation_date = block.target.deprecation_date
787
+ time_spine = (
788
+ TimeSpine(
789
+ standard_granularity_column=block.target.time_spine.standard_granularity_column,
790
+ custom_granularities=[
791
+ CustomGranularity(
792
+ name=custom_granularity.name,
793
+ column_name=custom_granularity.column_name,
794
+ )
795
+ for custom_granularity in block.target.time_spine.custom_granularities
796
+ ],
797
+ )
798
+ if block.target.time_spine
799
+ else None
800
+ )
801
+
802
+ return ParsedNodePatch(
803
+ name=block.target.name,
804
+ original_file_path=block.target.original_file_path,
805
+ yaml_key=block.target.yaml_key,
806
+ package_name=block.target.package_name,
807
+ description=block.target.description,
808
+ columns=refs.column_info,
809
+ meta=block.target.meta,
810
+ docs=block.target.docs,
811
+ config=block.target.config,
812
+ access=block.target.access,
813
+ version=None,
814
+ latest_version=None,
815
+ constraints=block.target.constraints,
816
+ deprecation_date=deprecation_date,
817
+ time_spine=time_spine,
818
+ )
819
+
820
+ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None:
821
+ patch = self._get_node_patch(block, refs)
822
+
823
+ assert isinstance(self.yaml.file, SchemaSourceFile)
824
+ source_file: SchemaSourceFile = self.yaml.file
825
+
826
+ # TODO: I'd like to refactor this out but the early return makes doing so a bit messy
827
+ if patch.yaml_key in ["models", "seeds", "snapshots"]:
828
+ unique_id = self.manifest.ref_lookup.get_unique_id(
829
+ patch.name, self.project.project_name, None
830
+ ) or self.manifest.ref_lookup.get_unique_id(patch.name, None, None)
831
+
832
+ if unique_id:
833
+ resource_type = NodeType(unique_id.split(".")[0])
834
+ if resource_type.pluralize() != patch.yaml_key:
835
+ warn_or_error(
836
+ WrongResourceSchemaFile(
837
+ patch_name=patch.name,
838
+ resource_type=resource_type,
839
+ plural_resource_type=resource_type.pluralize(),
840
+ yaml_key=patch.yaml_key,
841
+ file_path=patch.original_file_path,
842
+ )
843
+ )
844
+ return
845
+ elif patch.yaml_key == "functions":
846
+ unique_id = self.manifest.function_lookup.get_unique_id(patch.name, None)
847
+ elif patch.yaml_key == "analyses":
848
+ unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None, None)
849
+ else:
850
+ raise DbtInternalError(
851
+ f"Unexpected yaml_key {patch.yaml_key} for patch in "
852
+ f"file {source_file.path.original_file_path}"
853
+ )
854
+ # handle disabled nodes
855
+ if unique_id is None:
856
+ # Node might be disabled. Following call returns list of matching disabled nodes
857
+ resource_type = schema_file_keys_to_resource_types[patch.yaml_key]
858
+ found_nodes = self.manifest.disabled_lookup.find(
859
+ patch.name, patch.package_name, resource_types=[resource_type]
860
+ )
861
+ if found_nodes:
862
+ if len(found_nodes) > 1 and patch.config.get("enabled"):
863
+ # There are multiple disabled nodes for this model and the schema file wants to enable one.
864
+ # We have no way to know which one to enable.
865
+ resource_type = found_nodes[0].unique_id.split(".")[0]
866
+ msg = (
867
+ f"Found {len(found_nodes)} matching disabled nodes for "
868
+ f"{resource_type} '{patch.name}'. Multiple nodes for the same "
869
+ "unique id cannot be enabled in the schema file. They must be enabled "
870
+ "in `dbt_project.yml` or in the sql files."
871
+ )
872
+ raise ParsingError(msg)
873
+
874
+ # all nodes in the disabled dict have the same unique_id so just grab the first one
875
+ # to append with the unique id
876
+ source_file.append_patch(patch.yaml_key, found_nodes[0].unique_id)
877
+ for node in found_nodes:
878
+ node.patch_path = source_file.file_id
879
+ # re-calculate the node config with the patch config. Always do this
880
+ # for the case when no config is set to ensure the default of true gets captured
881
+ if patch.config:
882
+ self.patch_node_config(node, patch)
883
+
884
+ self.patch_node_properties(node, patch)
885
+ else:
886
+ warn_or_error(
887
+ NoNodeForYamlKey(
888
+ patch_name=patch.name,
889
+ yaml_key=patch.yaml_key,
890
+ file_path=source_file.path.original_file_path,
891
+ )
892
+ )
893
+ return # we only return early if no disabled early nodes are found. Why don't we return after patching the disabled nodes?
894
+
895
+ if patch.yaml_key == "functions":
896
+ node = self.manifest.functions.get(unique_id)
897
+ else:
898
+ node = self.manifest.nodes.get(unique_id)
899
+
900
+ if node:
901
+ # patches can't be overwritten
902
+ if node.patch_path:
903
+ package_name, existing_file_path = node.patch_path.split("://")
904
+ raise DuplicatePatchPathError(patch, existing_file_path)
905
+
906
+ source_file.append_patch(patch.yaml_key, node.unique_id)
907
+ # re-calculate the node config with the patch config. Always do this
908
+ # for the case when no config is set to ensure the default of true gets captured
909
+ if patch.config:
910
+ self.patch_node_config(node, patch)
911
+
912
+ self.patch_node_properties(node, patch)
913
+
914
+ def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None:
915
+ """Given a ParsedNodePatch, add the new information to the node."""
916
+ # explicitly pick out the parts to update so we don't inadvertently
917
+ # step on the model name or anything
918
+ # Note: config should already be updated
919
+ node.patch_path = patch.file_id
920
+ # update created_at so process_docs will run in partial parsing
921
+ node.created_at = time.time()
922
+ node.description = patch.description
923
+ node.columns = patch.columns
924
+ node.name = patch.name
925
+
926
+ if not isinstance(node, ModelNode):
927
+ for attr in ["latest_version", "access", "version", "constraints"]:
928
+ if getattr(patch, attr):
929
+ warn_or_error(
930
+ ValidationWarning(
931
+ field_name=attr,
932
+ resource_type=node.resource_type.value,
933
+ node_name=patch.name,
934
+ )
935
+ )
936
+
937
+
938
+ # TestablePatchParser = seeds, snapshots
939
+ class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]):
940
+ __test__ = False
941
+
942
+ def get_block(self, node: UnparsedNodeUpdate) -> TestBlock:
943
+ return TestBlock.from_yaml_block(self.yaml, node)
944
+
945
+ def _target_type(self) -> Type[UnparsedNodeUpdate]:
946
+ return UnparsedNodeUpdate
947
+
948
+
949
+ class ModelPatchParser(NodePatchParser[UnparsedModelUpdate]):
950
+ def get_block(self, node: UnparsedModelUpdate) -> VersionedTestBlock:
951
+ return VersionedTestBlock.from_yaml_block(self.yaml, node)
952
+
953
+ def parse_patch(self, block: TargetBlock[UnparsedModelUpdate], refs: ParserRef) -> None:
954
+ target = block.target
955
+ if NodeType.Model.pluralize() != target.yaml_key:
956
+ warn_or_error(
957
+ WrongResourceSchemaFile(
958
+ patch_name=target.name,
959
+ resource_type=NodeType.Model,
960
+ plural_resource_type=NodeType.Model.pluralize(),
961
+ yaml_key=target.yaml_key,
962
+ file_path=target.original_file_path,
963
+ )
964
+ )
965
+ return
966
+
967
+ versions = target.versions
968
+ if not versions:
969
+ super().parse_patch(block, refs)
970
+ else:
971
+ assert isinstance(self.yaml.file, SchemaSourceFile)
972
+ source_file: SchemaSourceFile = self.yaml.file
973
+ latest_version = (
974
+ target.latest_version if target.latest_version is not None else max(versions).v
975
+ )
976
+ for unparsed_version in versions:
977
+ versioned_model_name = (
978
+ unparsed_version.defined_in or f"{block.name}_{unparsed_version.formatted_v}"
979
+ )
980
+ # ref lookup without version - version is not set yet
981
+ versioned_model_unique_id = self.manifest.ref_lookup.get_unique_id(
982
+ versioned_model_name, target.package_name, None
983
+ )
984
+
985
+ versioned_model_node: Optional[ModelNode] = None
986
+ add_node_nofile_fn: Callable
987
+
988
+ # If this is the latest version, it's allowed to define itself in a model file name that doesn't have a suffix
989
+ if versioned_model_unique_id is None and unparsed_version.v == latest_version:
990
+ versioned_model_unique_id = self.manifest.ref_lookup.get_unique_id(
991
+ block.name, target.package_name, None
992
+ )
993
+
994
+ if versioned_model_unique_id is None:
995
+ # Node might be disabled. Following call returns list of matching disabled nodes
996
+ found_nodes = self.manifest.disabled_lookup.find(
997
+ versioned_model_name, None, resource_types=[NodeType.Model]
998
+ )
999
+ if found_nodes:
1000
+ if len(found_nodes) > 1 and target.config.get("enabled"):
1001
+ # There are multiple disabled nodes for this model and the schema file wants to enable one.
1002
+ # We have no way to know which one to enable.
1003
+ resource_type = found_nodes[0].unique_id.split(".")[0]
1004
+ msg = (
1005
+ f"Found {len(found_nodes)} matching disabled nodes for "
1006
+ f"{resource_type} '{target.name}'. Multiple nodes for the same "
1007
+ "unique id cannot be enabled in the schema file. They must be enabled "
1008
+ "in `dbt_project.yml` or in the sql files."
1009
+ )
1010
+ raise ParsingError(msg)
1011
+ # We know that there's only one node in the disabled list because
1012
+ # otherwise we would have raised the error above
1013
+ found_node = found_nodes[0]
1014
+ self.manifest.disabled.pop(found_node.unique_id)
1015
+ assert isinstance(found_node, ModelNode)
1016
+ versioned_model_node = found_node
1017
+ add_node_nofile_fn = self.manifest.add_disabled_nofile
1018
+ else:
1019
+ found_node = self.manifest.nodes.pop(versioned_model_unique_id)
1020
+ assert isinstance(found_node, ModelNode)
1021
+ versioned_model_node = found_node
1022
+ add_node_nofile_fn = self.manifest.add_node_nofile
1023
+
1024
+ if versioned_model_node is None:
1025
+ warn_or_error(
1026
+ NoNodeForYamlKey(
1027
+ patch_name=versioned_model_name,
1028
+ yaml_key=target.yaml_key,
1029
+ file_path=source_file.path.original_file_path,
1030
+ )
1031
+ )
1032
+ continue
1033
+
1034
+ # update versioned node unique_id
1035
+ versioned_model_node_unique_id_old = versioned_model_node.unique_id
1036
+ versioned_model_node.unique_id = (
1037
+ f"model.{target.package_name}.{target.name}.{unparsed_version.formatted_v}"
1038
+ )
1039
+ # update source file.nodes with new unique_id
1040
+ model_source_file = self.manifest.files[versioned_model_node.file_id]
1041
+ assert isinstance(model_source_file, SourceFile)
1042
+ # because of incomplete test setup, check before removing
1043
+ if versioned_model_node_unique_id_old in model_source_file.nodes:
1044
+ model_source_file.nodes.remove(versioned_model_node_unique_id_old)
1045
+ model_source_file.nodes.append(versioned_model_node.unique_id)
1046
+
1047
+ # update versioned node fqn
1048
+ versioned_model_node.fqn[-1] = target.name
1049
+ versioned_model_node.fqn.append(unparsed_version.formatted_v)
1050
+
1051
+ # add versioned node back to nodes/disabled
1052
+ add_node_nofile_fn(versioned_model_node)
1053
+
1054
+ # flatten columns based on include/exclude
1055
+ version_refs: ParserRef = ParserRef.from_versioned_target(
1056
+ block.target, unparsed_version.v
1057
+ )
1058
+
1059
+ versioned_model_patch = ParsedNodePatch(
1060
+ name=target.name,
1061
+ original_file_path=target.original_file_path,
1062
+ yaml_key=target.yaml_key,
1063
+ package_name=target.package_name,
1064
+ description=unparsed_version.description or target.description,
1065
+ columns=version_refs.column_info,
1066
+ meta=target.meta,
1067
+ docs=unparsed_version.docs or target.docs,
1068
+ config=deep_merge(target.config, unparsed_version.config),
1069
+ access=unparsed_version.access or target.access,
1070
+ version=unparsed_version.v,
1071
+ latest_version=latest_version,
1072
+ constraints=unparsed_version.constraints or target.constraints,
1073
+ deprecation_date=unparsed_version.deprecation_date,
1074
+ )
1075
+ # Node patched before config because config patching depends on model name,
1076
+ # which may have been updated in the version patch
1077
+ # versioned_model_node.patch(versioned_model_patch)
1078
+ self.patch_node_properties(versioned_model_node, versioned_model_patch)
1079
+
1080
+ # Includes alias recomputation
1081
+ self.patch_node_config(versioned_model_node, versioned_model_patch)
1082
+
1083
+ # Need to reapply setting constraints and contract checksum here, because
1084
+ # they depend on node.contract.enabled, which wouldn't be set when
1085
+ # patch_node_properties was called if it wasn't set in the model file.
1086
+ self.patch_constraints(versioned_model_node, versioned_model_patch.constraints)
1087
+ versioned_model_node.build_contract_checksum()
1088
+ source_file.append_patch(
1089
+ versioned_model_patch.yaml_key, versioned_model_node.unique_id
1090
+ )
1091
+ self.manifest.rebuild_ref_lookup()
1092
+ self.manifest.rebuild_disabled_lookup()
1093
+
1094
+ def _target_type(self) -> Type[UnparsedModelUpdate]:
1095
+ return UnparsedModelUpdate
1096
+
1097
+ def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None:
1098
+ super().patch_node_properties(node, patch)
1099
+
1100
+ # Remaining patch properties are only relevant to ModelNode objects
1101
+ if not isinstance(node, ModelNode):
1102
+ return
1103
+
1104
+ node.version = patch.version
1105
+ node.latest_version = patch.latest_version
1106
+ node.deprecation_date = patch.deprecation_date
1107
+ if patch.access:
1108
+ if AccessType.is_valid(patch.access):
1109
+ node.access = AccessType(patch.access)
1110
+ else:
1111
+ raise InvalidAccessTypeError(
1112
+ unique_id=node.unique_id,
1113
+ field_value=patch.access,
1114
+ )
1115
+ # These two will have to be reapplied after config is built for versioned models
1116
+ self.patch_constraints(node, patch.constraints)
1117
+ self.patch_time_spine(node, patch.time_spine)
1118
+ node.build_contract_checksum()
1119
+
1120
+ def patch_constraints(self, node: ModelNode, constraints: List[Dict[str, Any]]) -> None:
1121
+ contract_config = node.config.get("contract")
1122
+ if contract_config.enforced is True:
1123
+ self._validate_constraint_prerequisites(node)
1124
+
1125
+ if any(
1126
+ c for c in constraints if "type" not in c or not ConstraintType.is_valid(c["type"])
1127
+ ):
1128
+ raise ParsingError(
1129
+ f"Invalid constraint type on model {node.name}: "
1130
+ f"Type must be one of {[ct.value for ct in ConstraintType]}"
1131
+ )
1132
+
1133
+ self._validate_pk_constraints(node, constraints)
1134
+ node.constraints = [ModelLevelConstraint.from_dict(c) for c in constraints]
1135
+ self._process_constraints_refs_and_sources(node)
1136
+
1137
+ def _process_constraints_refs_and_sources(self, model_node: ModelNode) -> None:
1138
+ """
1139
+ Populate model_node.refs and model_node.sources based on foreign-key constraint references,
1140
+ whether defined at the model-level or column-level.
1141
+ """
1142
+ for constraint in model_node.all_constraints:
1143
+ if constraint.type == ConstraintType.foreign_key and constraint.to:
1144
+ try:
1145
+ ref_or_source = statically_parse_ref_or_source(constraint.to)
1146
+ except ParsingError:
1147
+ raise ParsingError(
1148
+ f"Invalid 'ref' or 'source' syntax on foreign key constraint 'to' on model {model_node.name}: {constraint.to}."
1149
+ )
1150
+
1151
+ if isinstance(ref_or_source, RefArgs):
1152
+ model_node.refs.append(ref_or_source)
1153
+ else:
1154
+ model_node.sources.append(ref_or_source)
1155
+
1156
+ def patch_time_spine(self, node: ModelNode, time_spine: Optional[TimeSpine]) -> None:
1157
+ node.time_spine = time_spine
1158
+
1159
+ def _validate_pk_constraints(
1160
+ self, model_node: ModelNode, constraints: List[Dict[str, Any]]
1161
+ ) -> None:
1162
+ errors = []
1163
+ # check for primary key constraints defined at the column level
1164
+ pk_col: List[str] = []
1165
+ for col in model_node.columns.values():
1166
+ for constraint in col.constraints:
1167
+ if constraint.type == ConstraintType.primary_key:
1168
+ pk_col.append(col.name)
1169
+
1170
+ if len(pk_col) > 1:
1171
+ errors.append(
1172
+ f"Found {len(pk_col)} columns ({pk_col}) with primary key constraints defined. "
1173
+ "Primary keys for multiple columns must be defined as a model level constraint."
1174
+ )
1175
+
1176
+ if len(pk_col) > 0 and (
1177
+ any(
1178
+ constraint.type == ConstraintType.primary_key
1179
+ for constraint in model_node.constraints
1180
+ )
1181
+ or any(constraint["type"] == ConstraintType.primary_key for constraint in constraints)
1182
+ ):
1183
+ errors.append(
1184
+ "Primary key constraints defined at the model level and the columns level. "
1185
+ "Primary keys can be defined at the model level or the column level, not both."
1186
+ )
1187
+
1188
+ if errors:
1189
+ raise ParsingError(
1190
+ f"Primary key constraint error: ({model_node.original_file_path})\n"
1191
+ + "\n".join(errors)
1192
+ )
1193
+
1194
+ def _validate_constraint_prerequisites(self, model_node: ModelNode) -> None:
1195
+ column_warn_unsupported = [
1196
+ constraint.warn_unsupported
1197
+ for column in model_node.columns.values()
1198
+ for constraint in column.constraints
1199
+ ]
1200
+ model_warn_unsupported = [
1201
+ constraint.warn_unsupported for constraint in model_node.constraints
1202
+ ]
1203
+ warn_unsupported = column_warn_unsupported + model_warn_unsupported
1204
+
1205
+ # if any constraint has `warn_unsupported` as True then send the warning
1206
+ if any(warn_unsupported) and not model_node.materialization_enforces_constraints:
1207
+ warn_or_error(
1208
+ UnsupportedConstraintMaterialization(materialized=model_node.config.materialized),
1209
+ node=model_node,
1210
+ )
1211
+
1212
+ errors = []
1213
+ if not model_node.columns:
1214
+ errors.append(
1215
+ "Constraints must be defined in a `yml` schema configuration file like `schema.yml`."
1216
+ )
1217
+
1218
+ if str(model_node.language) != "sql":
1219
+ errors.append(f"Language Error: Expected 'sql' but found '{model_node.language}'")
1220
+
1221
+ if errors:
1222
+ raise ParsingError(
1223
+ f"Contract enforcement failed for: ({model_node.original_file_path})\n"
1224
+ + "\n".join(errors)
1225
+ )
1226
+
1227
+
1228
+ class AnalysisPatchParser(NodePatchParser[UnparsedAnalysisUpdate]):
1229
+ def get_block(self, node: UnparsedAnalysisUpdate) -> TargetBlock:
1230
+ return TargetBlock.from_yaml_block(self.yaml, node)
1231
+
1232
+ def _target_type(self) -> Type[UnparsedAnalysisUpdate]:
1233
+ return UnparsedAnalysisUpdate
1234
+
1235
+
1236
+ class SingularTestPatchParser(PatchParser[UnparsedSingularTestUpdate, ParsedSingularTestPatch]):
1237
+ def get_block(self, node: UnparsedSingularTestUpdate) -> TargetBlock:
1238
+ return TargetBlock.from_yaml_block(self.yaml, node)
1239
+
1240
+ def _target_type(self) -> Type[UnparsedSingularTestUpdate]:
1241
+ return UnparsedSingularTestUpdate
1242
+
1243
+ def parse_patch(self, block: TargetBlock[UnparsedSingularTestUpdate], refs: ParserRef) -> None:
1244
+ patch = ParsedSingularTestPatch(
1245
+ name=block.target.name,
1246
+ description=block.target.description,
1247
+ meta=block.target.meta,
1248
+ docs=block.target.docs,
1249
+ config=block.target.config,
1250
+ original_file_path=block.target.original_file_path,
1251
+ yaml_key=block.target.yaml_key,
1252
+ package_name=block.target.package_name,
1253
+ )
1254
+
1255
+ assert isinstance(self.yaml.file, SchemaSourceFile)
1256
+ source_file: SchemaSourceFile = self.yaml.file
1257
+
1258
+ unique_id = self.manifest.singular_test_lookup.get_unique_id(
1259
+ block.name, block.target.package_name
1260
+ )
1261
+ if not unique_id:
1262
+ warn_or_error(
1263
+ NoNodeForYamlKey(
1264
+ patch_name=patch.name,
1265
+ yaml_key=patch.yaml_key,
1266
+ file_path=source_file.path.original_file_path,
1267
+ )
1268
+ )
1269
+ return
1270
+
1271
+ node = self.manifest.nodes.get(unique_id)
1272
+ assert node is not None
1273
+
1274
+ source_file.append_patch(patch.yaml_key, unique_id)
1275
+ if patch.config:
1276
+ self.patch_node_config(node, patch)
1277
+
1278
+ node.patch_path = patch.file_id
1279
+ node.description = patch.description
1280
+ node.created_at = time.time()
1281
+
1282
+
1283
+ class FunctionPatchParser(NodePatchParser[UnparsedFunctionUpdate]):
1284
+ def get_block(self, node: UnparsedFunctionUpdate) -> TargetBlock:
1285
+ return TargetBlock.from_yaml_block(self.yaml, node)
1286
+
1287
+ def _target_type(self) -> Type[UnparsedFunctionUpdate]:
1288
+ return UnparsedFunctionUpdate
1289
+
1290
+ def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None:
1291
+ super().patch_node_properties(node, patch)
1292
+
1293
+ assert isinstance(patch, ParsedFunctionPatch)
1294
+ assert isinstance(node, FunctionNode)
1295
+
1296
+ node.arguments = patch.arguments
1297
+ node.returns = patch.returns
1298
+
1299
+ def _get_node_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> ParsedNodePatch:
1300
+ target = block.target
1301
+ assert isinstance(target, UnparsedFunctionUpdate)
1302
+
1303
+ return ParsedFunctionPatch(
1304
+ name=target.name,
1305
+ original_file_path=target.original_file_path,
1306
+ yaml_key=target.yaml_key,
1307
+ package_name=target.package_name,
1308
+ description=target.description,
1309
+ columns=refs.column_info,
1310
+ meta=target.meta,
1311
+ docs=target.docs,
1312
+ config=target.config,
1313
+ access=target.access,
1314
+ version=None,
1315
+ latest_version=None,
1316
+ constraints=target.constraints,
1317
+ deprecation_date=None,
1318
+ time_spine=None,
1319
+ arguments=target.arguments,
1320
+ returns=target.returns,
1321
+ )
1322
+
1323
+
1324
+ class MacroPatchParser(PatchParser[UnparsedMacroUpdate, ParsedMacroPatch]):
1325
+ def get_block(self, node: UnparsedMacroUpdate) -> TargetBlock:
1326
+ return TargetBlock.from_yaml_block(self.yaml, node)
1327
+
1328
+ def _target_type(self) -> Type[UnparsedMacroUpdate]:
1329
+ return UnparsedMacroUpdate
1330
+
1331
+ def parse_patch(self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef) -> None:
1332
+ patch = ParsedMacroPatch(
1333
+ name=block.target.name,
1334
+ original_file_path=block.target.original_file_path,
1335
+ yaml_key=block.target.yaml_key,
1336
+ package_name=block.target.package_name,
1337
+ arguments=block.target.arguments,
1338
+ description=block.target.description,
1339
+ meta=block.target.meta,
1340
+ docs=block.target.docs,
1341
+ config=block.target.config,
1342
+ )
1343
+ assert isinstance(self.yaml.file, SchemaSourceFile)
1344
+ source_file = self.yaml.file
1345
+ # macros are fully namespaced
1346
+ unique_id = f"macro.{patch.package_name}.{patch.name}"
1347
+ macro = self.manifest.macros.get(unique_id)
1348
+ if not macro:
1349
+ warn_or_error(MacroNotFoundForPatch(patch_name=patch.name))
1350
+ return
1351
+ if macro.patch_path:
1352
+ package_name, existing_file_path = macro.patch_path.split("://")
1353
+ raise DuplicateMacroPatchNameError(patch, existing_file_path)
1354
+ source_file.macro_patches[patch.name] = unique_id
1355
+
1356
+ # former macro.patch code
1357
+ macro.patch_path = patch.file_id
1358
+ macro.description = patch.description
1359
+ macro.created_at = time.time()
1360
+ macro.meta = patch.meta
1361
+ macro.docs = patch.docs
1362
+
1363
+ if getattr(get_flags(), "validate_macro_args", False):
1364
+ self._check_patch_arguments(macro, patch)
1365
+ macro.arguments = patch.arguments if patch.arguments else macro.arguments
1366
+ else:
1367
+ macro.arguments = patch.arguments
1368
+
1369
+ def _check_patch_arguments(self, macro: Macro, patch: ParsedMacroPatch) -> None:
1370
+ if not patch.arguments:
1371
+ return
1372
+
1373
+ for macro_arg, patch_arg in zip(macro.arguments, patch.arguments):
1374
+ if patch_arg.name != macro_arg.name:
1375
+ msg = f"Argument {patch_arg.name} in yaml for macro {macro.name} does not match the jinja definition."
1376
+ self._fire_macro_arg_warning(msg, macro)
1377
+
1378
+ if len(patch.arguments) != len(macro.arguments):
1379
+ msg = f"The number of arguments in the yaml for macro {macro.name} does not match the jinja definition."
1380
+ self._fire_macro_arg_warning(msg, macro)
1381
+
1382
+ for patch_arg in patch.arguments:
1383
+ arg_type = patch_arg.type
1384
+ if arg_type is not None and arg_type.strip() != "" and not is_valid_type(arg_type):
1385
+ msg = f"Argument {patch_arg.name} in the yaml for macro {macro.name} has an invalid type."
1386
+ self._fire_macro_arg_warning(msg, macro)
1387
+
1388
+ def _fire_macro_arg_warning(self, msg: str, macro: Macro) -> None:
1389
+ warn_or_error(
1390
+ InvalidMacroAnnotation(
1391
+ msg=msg, macro_unique_id=macro.unique_id, macro_file_path=macro.original_file_path
1392
+ )
1393
+ )
1394
+
1395
+
1396
+ # valid type names, along with the number of parameters they require
1397
+ macro_types: Dict[str, int] = {
1398
+ "str": 0,
1399
+ "string": 0,
1400
+ "bool": 0,
1401
+ "int": 0,
1402
+ "integer": 0,
1403
+ "float": 0,
1404
+ "any": 0,
1405
+ "list": 1,
1406
+ "dict": 2,
1407
+ "optional": 1,
1408
+ "relation": 0,
1409
+ "column": 0,
1410
+ }
1411
+
1412
+
1413
+ def is_valid_type(buffer: str) -> bool:
1414
+ buffer = buffer.replace(" ", "").replace("\t", "")
1415
+ type_desc, remainder = match_type_desc(buffer)
1416
+ return type_desc is not None and remainder == ""
1417
+
1418
+
1419
+ def match_type_desc(buffer: str) -> Tuple[Optional[str], str]:
1420
+ """A matching buffer is a type name followed by an argument list with
1421
+ the correct number of arguments."""
1422
+ type_name, remainder = match_type_name(buffer)
1423
+ if type_name is None:
1424
+ return None, buffer
1425
+ attr_list, remainder = match_arg_list(remainder, macro_types[type_name])
1426
+ if attr_list is None:
1427
+ return None, buffer
1428
+ return type_name + attr_list, remainder
1429
+
1430
+
1431
+ alpha_pattern = re.compile(r"[a-z]+")
1432
+
1433
+
1434
+ def match_type_name(buffer: str) -> Tuple[Optional[str], str]:
1435
+ """A matching buffer starts with one of the valid type names from macro_types"""
1436
+ match = alpha_pattern.match(buffer)
1437
+ if match is not None and buffer[: match.end(0)] in macro_types:
1438
+ return buffer[: match.end(0)], buffer[match.end(0) :]
1439
+ else:
1440
+ return None, buffer
1441
+
1442
+
1443
+ def match_arg_list(buffer: str, arg_count: int) -> Tuple[Optional[str], str]:
1444
+ """A matching buffer must begin with '[', followed by exactly arg_count type
1445
+ specs, followed by ']'"""
1446
+
1447
+ if arg_count == 0:
1448
+ return "", buffer
1449
+
1450
+ if not buffer.startswith("["):
1451
+ return None, buffer
1452
+
1453
+ remainder = buffer[1:]
1454
+ for i in range(arg_count):
1455
+ type_desc, remainder = match_type_desc(remainder)
1456
+ if type_desc is None:
1457
+ return None, buffer
1458
+ if i != arg_count - 1:
1459
+ if not remainder.startswith(","):
1460
+ return None, buffer
1461
+ remainder = remainder[1:]
1462
+
1463
+ if not remainder.startswith("]"):
1464
+ return None, buffer
1465
+ else:
1466
+ return "", remainder[1:]