dbt-bouncer 1.31.2rc3__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_bouncer/artifact_parsers/dbt_cloud/catalog_latest.py +21 -21
- dbt_bouncer/artifact_parsers/dbt_cloud/manifest_latest.py +1745 -1745
- dbt_bouncer/artifact_parsers/dbt_cloud/run_results_latest.py +22 -22
- dbt_bouncer/artifact_parsers/parsers_catalog.py +26 -24
- dbt_bouncer/artifact_parsers/parsers_common.py +57 -36
- dbt_bouncer/artifact_parsers/parsers_manifest.py +98 -69
- dbt_bouncer/artifact_parsers/parsers_run_results.py +32 -19
- dbt_bouncer/check_base.py +22 -11
- dbt_bouncer/checks/catalog/check_catalog_sources.py +22 -12
- dbt_bouncer/checks/catalog/check_columns.py +175 -105
- dbt_bouncer/checks/common.py +24 -3
- dbt_bouncer/checks/manifest/check_exposures.py +79 -52
- dbt_bouncer/checks/manifest/check_lineage.py +69 -40
- dbt_bouncer/checks/manifest/check_macros.py +177 -104
- dbt_bouncer/checks/manifest/check_metadata.py +28 -18
- dbt_bouncer/checks/manifest/check_models.py +842 -496
- dbt_bouncer/checks/manifest/check_seeds.py +63 -0
- dbt_bouncer/checks/manifest/check_semantic_models.py +28 -20
- dbt_bouncer/checks/manifest/check_snapshots.py +57 -33
- dbt_bouncer/checks/manifest/check_sources.py +246 -137
- dbt_bouncer/checks/manifest/check_unit_tests.py +97 -54
- dbt_bouncer/checks/run_results/check_run_results.py +34 -20
- dbt_bouncer/config_file_parser.py +47 -28
- dbt_bouncer/config_file_validator.py +11 -8
- dbt_bouncer/global_context.py +31 -0
- dbt_bouncer/main.py +128 -67
- dbt_bouncer/runner.py +61 -31
- dbt_bouncer/utils.py +146 -50
- dbt_bouncer/version.py +1 -1
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/METADATA +15 -15
- dbt_bouncer-2.0.0.dist-info/RECORD +37 -0
- dbt_bouncer-1.31.2rc3.dist-info/RECORD +0 -35
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/WHEEL +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/entry_points.txt +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
# mypy: disable-error-code="union-attr"
|
|
2
|
-
|
|
3
1
|
import re
|
|
4
|
-
from
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import TYPE_CHECKING, Literal
|
|
5
4
|
|
|
6
5
|
if TYPE_CHECKING:
|
|
7
|
-
from dbt_bouncer.artifact_parsers.
|
|
6
|
+
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
8
7
|
DbtBouncerModelBase,
|
|
9
8
|
DbtBouncerSourceBase,
|
|
10
9
|
)
|
|
@@ -14,37 +13,57 @@ if TYPE_CHECKING:
|
|
|
14
13
|
from pydantic import Field
|
|
15
14
|
|
|
16
15
|
from dbt_bouncer.check_base import BaseCheck
|
|
16
|
+
from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
|
|
17
17
|
from dbt_bouncer.utils import clean_path_str, find_missing_meta_keys
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class CheckSourceDescriptionPopulated(BaseCheck):
|
|
21
21
|
"""Sources must have a populated description.
|
|
22
22
|
|
|
23
|
+
Parameters:
|
|
24
|
+
min_description_length (int | None): Minimum length required for the description to be considered populated.
|
|
25
|
+
|
|
23
26
|
Receives:
|
|
24
27
|
source (DbtBouncerSourceBase): The DbtBouncerSourceBase object to check.
|
|
25
28
|
|
|
26
29
|
Other Parameters:
|
|
27
|
-
description (
|
|
28
|
-
exclude (
|
|
29
|
-
include (
|
|
30
|
-
severity (
|
|
30
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
31
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
32
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
33
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
31
34
|
|
|
32
35
|
Example(s):
|
|
33
36
|
```yaml
|
|
34
37
|
manifest_checks:
|
|
35
38
|
- name: check_source_description_populated
|
|
36
39
|
```
|
|
40
|
+
```yaml
|
|
41
|
+
manifest_checks:
|
|
42
|
+
- name: check_source_description_populated
|
|
43
|
+
min_description_length: 25 # Setting a stricter requirement for description length
|
|
44
|
+
```
|
|
37
45
|
|
|
38
46
|
"""
|
|
39
47
|
|
|
48
|
+
min_description_length: int | None = Field(default=None)
|
|
40
49
|
name: Literal["check_source_description_populated"]
|
|
41
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
50
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
42
51
|
|
|
43
52
|
def execute(self) -> None:
|
|
44
|
-
"""Execute the check.
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
53
|
+
"""Execute the check.
|
|
54
|
+
|
|
55
|
+
Raises:
|
|
56
|
+
DbtBouncerFailedCheckError: If description is not populated.
|
|
57
|
+
|
|
58
|
+
"""
|
|
59
|
+
if self.source is None:
|
|
60
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
61
|
+
if not self._is_description_populated(
|
|
62
|
+
self.source.description or "", self.min_description_length
|
|
63
|
+
):
|
|
64
|
+
raise DbtBouncerFailedCheckError(
|
|
65
|
+
f"`{self.source.source_name}.{self.source.name}` does not have a populated description."
|
|
66
|
+
)
|
|
48
67
|
|
|
49
68
|
|
|
50
69
|
class CheckSourceFreshnessPopulated(BaseCheck):
|
|
@@ -54,10 +73,10 @@ class CheckSourceFreshnessPopulated(BaseCheck):
|
|
|
54
73
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
55
74
|
|
|
56
75
|
Other Parameters:
|
|
57
|
-
description (
|
|
58
|
-
exclude (
|
|
59
|
-
include (
|
|
60
|
-
severity (
|
|
76
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
77
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
78
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
79
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
61
80
|
|
|
62
81
|
Example(s):
|
|
63
82
|
```yaml
|
|
@@ -68,19 +87,33 @@ class CheckSourceFreshnessPopulated(BaseCheck):
|
|
|
68
87
|
"""
|
|
69
88
|
|
|
70
89
|
name: Literal["check_source_freshness_populated"]
|
|
71
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
90
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
72
91
|
|
|
73
92
|
def execute(self) -> None:
|
|
74
|
-
"""Execute the check.
|
|
93
|
+
"""Execute the check.
|
|
94
|
+
|
|
95
|
+
Raises:
|
|
96
|
+
DbtBouncerFailedCheckError: If freshness is not populated.
|
|
97
|
+
|
|
98
|
+
"""
|
|
99
|
+
if self.source is None:
|
|
100
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
75
101
|
error_msg = f"`{self.source.source_name}.{self.source.name}` does not have a populated freshness."
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
102
|
+
if self.source.freshness is None:
|
|
103
|
+
raise DbtBouncerFailedCheckError(error_msg)
|
|
104
|
+
has_error_after = (
|
|
105
|
+
self.source.freshness.error_after
|
|
106
|
+
and self.source.freshness.error_after.count is not None
|
|
79
107
|
and self.source.freshness.error_after.period is not None
|
|
80
|
-
)
|
|
81
|
-
|
|
108
|
+
)
|
|
109
|
+
has_warn_after = (
|
|
110
|
+
self.source.freshness.warn_after
|
|
111
|
+
and self.source.freshness.warn_after.count is not None
|
|
82
112
|
and self.source.freshness.warn_after.period is not None
|
|
83
|
-
)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
if not (has_error_after or has_warn_after):
|
|
116
|
+
raise DbtBouncerFailedCheckError(error_msg)
|
|
84
117
|
|
|
85
118
|
|
|
86
119
|
class CheckSourceHasMetaKeys(BaseCheck):
|
|
@@ -93,10 +126,10 @@ class CheckSourceHasMetaKeys(BaseCheck):
|
|
|
93
126
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
94
127
|
|
|
95
128
|
Other Parameters:
|
|
96
|
-
description (
|
|
97
|
-
exclude (
|
|
98
|
-
include (
|
|
99
|
-
severity (
|
|
129
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
130
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
131
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
132
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
100
133
|
|
|
101
134
|
Example(s):
|
|
102
135
|
```yaml
|
|
@@ -113,33 +146,41 @@ class CheckSourceHasMetaKeys(BaseCheck):
|
|
|
113
146
|
|
|
114
147
|
keys: "NestedDict"
|
|
115
148
|
name: Literal["check_source_has_meta_keys"]
|
|
116
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
149
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
117
150
|
|
|
118
151
|
def execute(self) -> None:
|
|
119
|
-
"""Execute the check.
|
|
152
|
+
"""Execute the check.
|
|
153
|
+
|
|
154
|
+
Raises:
|
|
155
|
+
DbtBouncerFailedCheckError: If required meta keys are missing.
|
|
156
|
+
|
|
157
|
+
"""
|
|
158
|
+
if self.source is None:
|
|
159
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
120
160
|
missing_keys = find_missing_meta_keys(
|
|
121
161
|
meta_config=self.source.meta,
|
|
122
162
|
required_keys=self.keys.model_dump(),
|
|
123
163
|
)
|
|
124
164
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
165
|
+
if missing_keys != []:
|
|
166
|
+
raise DbtBouncerFailedCheckError(
|
|
167
|
+
f"`{self.source.source_name}.{self.source.name}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}"
|
|
168
|
+
)
|
|
128
169
|
|
|
129
170
|
|
|
130
171
|
class CheckSourceHasTags(BaseCheck):
|
|
131
172
|
"""Sources must have the specified tags.
|
|
132
173
|
|
|
133
174
|
Parameters:
|
|
134
|
-
criteria: (
|
|
175
|
+
criteria: (Literal["any", "all", "one"] | None): Whether the source must have any, all, or exactly one of the specified tags. Default: `all`.
|
|
135
176
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
136
|
-
tags (
|
|
177
|
+
tags (list[str]): List of tags to check for.
|
|
137
178
|
|
|
138
179
|
Other Parameters:
|
|
139
|
-
description (
|
|
140
|
-
exclude (
|
|
141
|
-
include (
|
|
142
|
-
severity (
|
|
180
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
181
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
182
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
183
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
143
184
|
|
|
144
185
|
Example(s):
|
|
145
186
|
```yaml
|
|
@@ -154,22 +195,34 @@ class CheckSourceHasTags(BaseCheck):
|
|
|
154
195
|
|
|
155
196
|
criteria: Literal["any", "all", "one"] = Field(default="all")
|
|
156
197
|
name: Literal["check_source_has_tags"]
|
|
157
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
158
|
-
tags:
|
|
198
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
199
|
+
tags: list[str]
|
|
159
200
|
|
|
160
201
|
def execute(self) -> None:
|
|
161
|
-
"""Execute the check.
|
|
202
|
+
"""Execute the check.
|
|
203
|
+
|
|
204
|
+
Raises:
|
|
205
|
+
DbtBouncerFailedCheckError: If source does not have required tags.
|
|
206
|
+
|
|
207
|
+
"""
|
|
208
|
+
if self.source is None:
|
|
209
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
210
|
+
source_tags = self.source.tags or []
|
|
162
211
|
if self.criteria == "any":
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
212
|
+
if not any(tag in source_tags for tag in self.tags):
|
|
213
|
+
raise DbtBouncerFailedCheckError(
|
|
214
|
+
f"`{self.source.source_name}.{self.source.name}` does not have any of the required tags: {self.tags}."
|
|
215
|
+
)
|
|
166
216
|
elif self.criteria == "all":
|
|
167
|
-
missing_tags =
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
217
|
+
missing_tags = set(self.tags) - set(source_tags)
|
|
218
|
+
if missing_tags:
|
|
219
|
+
raise DbtBouncerFailedCheckError(
|
|
220
|
+
f"`{self.source.source_name}.{self.source.name}` is missing required tags: {missing_tags}."
|
|
221
|
+
)
|
|
222
|
+
elif (
|
|
223
|
+
self.criteria == "one" and sum(tag in source_tags for tag in self.tags) != 1
|
|
224
|
+
):
|
|
225
|
+
raise DbtBouncerFailedCheckError(
|
|
173
226
|
f"`{self.source.source_name}.{self.source.name}` must have exactly one of the required tags: {self.tags}."
|
|
174
227
|
)
|
|
175
228
|
|
|
@@ -181,10 +234,10 @@ class CheckSourceLoaderPopulated(BaseCheck):
|
|
|
181
234
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
182
235
|
|
|
183
236
|
Other Parameters:
|
|
184
|
-
description (
|
|
185
|
-
exclude (
|
|
186
|
-
include (
|
|
187
|
-
severity (
|
|
237
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
238
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
239
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
240
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
188
241
|
|
|
189
242
|
Example(s):
|
|
190
243
|
```yaml
|
|
@@ -195,13 +248,21 @@ class CheckSourceLoaderPopulated(BaseCheck):
|
|
|
195
248
|
"""
|
|
196
249
|
|
|
197
250
|
name: Literal["check_source_loader_populated"]
|
|
198
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
251
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
199
252
|
|
|
200
253
|
def execute(self) -> None:
|
|
201
|
-
"""Execute the check.
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
254
|
+
"""Execute the check.
|
|
255
|
+
|
|
256
|
+
Raises:
|
|
257
|
+
DbtBouncerFailedCheckError: If loader is not populated.
|
|
258
|
+
|
|
259
|
+
"""
|
|
260
|
+
if self.source is None:
|
|
261
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
262
|
+
if self.source.loader == "":
|
|
263
|
+
raise DbtBouncerFailedCheckError(
|
|
264
|
+
f"`{self.source.source_name}.{self.source.name}` does not have a populated loader."
|
|
265
|
+
)
|
|
205
266
|
|
|
206
267
|
|
|
207
268
|
class CheckSourceNames(BaseCheck):
|
|
@@ -214,10 +275,10 @@ class CheckSourceNames(BaseCheck):
|
|
|
214
275
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
215
276
|
|
|
216
277
|
Other Parameters:
|
|
217
|
-
description (
|
|
218
|
-
exclude (
|
|
219
|
-
include (
|
|
220
|
-
severity (
|
|
278
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
279
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
280
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
281
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
221
282
|
|
|
222
283
|
Example(s):
|
|
223
284
|
```yaml
|
|
@@ -231,30 +292,38 @@ class CheckSourceNames(BaseCheck):
|
|
|
231
292
|
|
|
232
293
|
name: Literal["check_source_names"]
|
|
233
294
|
source_name_pattern: str
|
|
234
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
295
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
235
296
|
|
|
236
297
|
def execute(self) -> None:
|
|
237
|
-
"""Execute the check.
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
298
|
+
"""Execute the check.
|
|
299
|
+
|
|
300
|
+
Raises:
|
|
301
|
+
DbtBouncerFailedCheckError: If source name does not match regex.
|
|
302
|
+
|
|
303
|
+
"""
|
|
304
|
+
if self.source is None:
|
|
305
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
306
|
+
if (
|
|
307
|
+
re.compile(self.source_name_pattern.strip()).match(str(self.source.name))
|
|
308
|
+
is None
|
|
309
|
+
):
|
|
310
|
+
raise DbtBouncerFailedCheckError(
|
|
311
|
+
f"`{self.source.source_name}.{self.source.name}` does not match the supplied regex `({self.source_name_pattern.strip()})`."
|
|
312
|
+
)
|
|
244
313
|
|
|
245
314
|
|
|
246
315
|
class CheckSourceNotOrphaned(BaseCheck):
|
|
247
316
|
"""Sources must be referenced in at least one model.
|
|
248
317
|
|
|
249
318
|
Receives:
|
|
250
|
-
models (
|
|
319
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
251
320
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
252
321
|
|
|
253
322
|
Other Parameters:
|
|
254
|
-
description (
|
|
255
|
-
exclude (
|
|
256
|
-
include (
|
|
257
|
-
severity (
|
|
323
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
324
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
325
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
326
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
258
327
|
|
|
259
328
|
Example(s):
|
|
260
329
|
```yaml
|
|
@@ -264,18 +333,28 @@ class CheckSourceNotOrphaned(BaseCheck):
|
|
|
264
333
|
|
|
265
334
|
"""
|
|
266
335
|
|
|
267
|
-
models:
|
|
336
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
268
337
|
name: Literal["check_source_not_orphaned"]
|
|
269
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
338
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
270
339
|
|
|
271
340
|
def execute(self) -> None:
|
|
272
|
-
"""Execute the check.
|
|
341
|
+
"""Execute the check.
|
|
342
|
+
|
|
343
|
+
Raises:
|
|
344
|
+
DbtBouncerFailedCheckError: If source is orphaned.
|
|
345
|
+
|
|
346
|
+
"""
|
|
347
|
+
if self.source is None:
|
|
348
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
273
349
|
num_refs = sum(
|
|
274
|
-
self.source.unique_id in model.depends_on
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
f"Source `{self.source.source_name}.{self.source.name}` is orphaned, i.e. not referenced by any model."
|
|
350
|
+
self.source.unique_id in getattr(model.depends_on, "nodes", [])
|
|
351
|
+
for model in self.models
|
|
352
|
+
if model.depends_on
|
|
278
353
|
)
|
|
354
|
+
if num_refs < 1:
|
|
355
|
+
raise DbtBouncerFailedCheckError(
|
|
356
|
+
f"Source `{self.source.source_name}.{self.source.name}` is orphaned, i.e. not referenced by any model."
|
|
357
|
+
)
|
|
279
358
|
|
|
280
359
|
|
|
281
360
|
class CheckSourcePropertyFileLocation(BaseCheck):
|
|
@@ -285,10 +364,10 @@ class CheckSourcePropertyFileLocation(BaseCheck):
|
|
|
285
364
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
286
365
|
|
|
287
366
|
Other Parameters:
|
|
288
|
-
description (
|
|
289
|
-
exclude (
|
|
290
|
-
include (
|
|
291
|
-
severity (
|
|
367
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
368
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
369
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
370
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
292
371
|
|
|
293
372
|
Example(s):
|
|
294
373
|
```yaml
|
|
@@ -299,46 +378,57 @@ class CheckSourcePropertyFileLocation(BaseCheck):
|
|
|
299
378
|
"""
|
|
300
379
|
|
|
301
380
|
name: Literal["check_source_property_file_location"]
|
|
302
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
381
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
303
382
|
|
|
304
383
|
def execute(self) -> None:
|
|
305
|
-
"""Execute the check.
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
"
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
)
|
|
384
|
+
"""Execute the check.
|
|
385
|
+
|
|
386
|
+
Raises:
|
|
387
|
+
DbtBouncerFailedCheckError: If property file location is incorrect.
|
|
388
|
+
|
|
389
|
+
"""
|
|
390
|
+
if self.source is None:
|
|
391
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
392
|
+
original_path = Path(clean_path_str(self.source.original_file_path))
|
|
393
|
+
|
|
394
|
+
if (
|
|
395
|
+
len(original_path.parts) > 2
|
|
396
|
+
and original_path.parts[0] == "models"
|
|
397
|
+
and original_path.parts[1] == "staging"
|
|
398
|
+
):
|
|
399
|
+
subdir_parts = original_path.parent.parts[2:]
|
|
400
|
+
else:
|
|
401
|
+
subdir_parts = original_path.parent.parts
|
|
402
|
+
|
|
403
|
+
expected_substring = "_" + "_".join(subdir_parts) if subdir_parts else ""
|
|
404
|
+
properties_yml_name = original_path.name
|
|
405
|
+
|
|
406
|
+
if not properties_yml_name.startswith("_"):
|
|
407
|
+
raise DbtBouncerFailedCheckError(
|
|
408
|
+
f"The properties file for `{self.source.source_name}.{self.source.name}` (`{properties_yml_name}`) does not start with an underscore."
|
|
409
|
+
)
|
|
410
|
+
if expected_substring not in properties_yml_name:
|
|
411
|
+
raise DbtBouncerFailedCheckError(
|
|
412
|
+
f"The properties file for `{self.source.source_name}.{self.source.name}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substring}`)."
|
|
413
|
+
)
|
|
414
|
+
if not properties_yml_name.endswith("__sources.yml"):
|
|
415
|
+
raise DbtBouncerFailedCheckError(
|
|
416
|
+
f"The properties file for `{self.source.source_name}.{self.source.name}` (`{properties_yml_name}`) does not end with `__sources.yml`."
|
|
417
|
+
)
|
|
328
418
|
|
|
329
419
|
|
|
330
420
|
class CheckSourceUsedByModelsInSameDirectory(BaseCheck):
|
|
331
421
|
"""Sources can only be referenced by models that are located in the same directory where the source is defined.
|
|
332
422
|
|
|
333
423
|
Parameters:
|
|
334
|
-
models (
|
|
424
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
335
425
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
336
426
|
|
|
337
427
|
Other Parameters:
|
|
338
|
-
description (
|
|
339
|
-
exclude (
|
|
340
|
-
include (
|
|
341
|
-
severity (
|
|
428
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
429
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
430
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
431
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
342
432
|
|
|
343
433
|
Example(s):
|
|
344
434
|
```yaml
|
|
@@ -348,38 +438,47 @@ class CheckSourceUsedByModelsInSameDirectory(BaseCheck):
|
|
|
348
438
|
|
|
349
439
|
"""
|
|
350
440
|
|
|
351
|
-
models:
|
|
441
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
352
442
|
name: Literal["check_source_used_by_models_in_same_directory"]
|
|
353
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
443
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
354
444
|
|
|
355
445
|
def execute(self) -> None:
|
|
356
|
-
"""Execute the check.
|
|
446
|
+
"""Execute the check.
|
|
447
|
+
|
|
448
|
+
Raises:
|
|
449
|
+
DbtBouncerFailedCheckError: If source is referenced by models in different directory.
|
|
450
|
+
|
|
451
|
+
"""
|
|
452
|
+
if self.source is None:
|
|
453
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
357
454
|
reffed_models_not_in_same_dir = []
|
|
358
455
|
for model in self.models:
|
|
359
456
|
if (
|
|
360
|
-
|
|
457
|
+
model.depends_on
|
|
458
|
+
and self.source.unique_id in getattr(model.depends_on, "nodes", [])
|
|
361
459
|
and model.original_file_path.split("/")[:-1]
|
|
362
460
|
!= self.source.original_file_path.split("/")[:-1]
|
|
363
461
|
):
|
|
364
462
|
reffed_models_not_in_same_dir.append(model.name)
|
|
365
463
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
464
|
+
if len(reffed_models_not_in_same_dir) != 0:
|
|
465
|
+
raise DbtBouncerFailedCheckError(
|
|
466
|
+
f"Source `{self.source.source_name}.{self.source.name}` is referenced by models defined in a different directory: {reffed_models_not_in_same_dir}"
|
|
467
|
+
)
|
|
369
468
|
|
|
370
469
|
|
|
371
470
|
class CheckSourceUsedByOnlyOneModel(BaseCheck):
|
|
372
471
|
"""Each source can be referenced by a maximum of one model.
|
|
373
472
|
|
|
374
473
|
Receives:
|
|
375
|
-
models (
|
|
474
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
376
475
|
source (DbtBouncerSource): The DbtBouncerSourceBase object to check.
|
|
377
476
|
|
|
378
477
|
Other Parameters:
|
|
379
|
-
description (
|
|
380
|
-
exclude (
|
|
381
|
-
include (
|
|
382
|
-
severity (
|
|
478
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
479
|
+
exclude (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Source paths that match the pattern will not be checked.
|
|
480
|
+
include (str | None): Regex pattern to match the source path (i.e the .yml file where the source is configured). Only source paths that match the pattern will be checked.
|
|
481
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
383
482
|
|
|
384
483
|
Example(s):
|
|
385
484
|
```yaml
|
|
@@ -389,15 +488,25 @@ class CheckSourceUsedByOnlyOneModel(BaseCheck):
|
|
|
389
488
|
|
|
390
489
|
"""
|
|
391
490
|
|
|
392
|
-
models:
|
|
491
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
393
492
|
name: Literal["check_source_used_by_only_one_model"]
|
|
394
|
-
source: "DbtBouncerSourceBase" = Field(default=None)
|
|
493
|
+
source: "DbtBouncerSourceBase | None" = Field(default=None)
|
|
395
494
|
|
|
396
495
|
def execute(self) -> None:
|
|
397
|
-
"""Execute the check.
|
|
496
|
+
"""Execute the check.
|
|
497
|
+
|
|
498
|
+
Raises:
|
|
499
|
+
DbtBouncerFailedCheckError: If source is referenced by more than one model.
|
|
500
|
+
|
|
501
|
+
"""
|
|
502
|
+
if self.source is None:
|
|
503
|
+
raise DbtBouncerFailedCheckError("self.source is None")
|
|
398
504
|
num_refs = sum(
|
|
399
|
-
self.source.unique_id in model.depends_on
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
f"Source `{self.source.source_name}.{self.source.name}` is referenced by more than one model."
|
|
505
|
+
self.source.unique_id in getattr(model.depends_on, "nodes", [])
|
|
506
|
+
for model in self.models
|
|
507
|
+
if model.depends_on
|
|
403
508
|
)
|
|
509
|
+
if num_refs > 1:
|
|
510
|
+
raise DbtBouncerFailedCheckError(
|
|
511
|
+
f"Source `{self.source.source_name}.{self.source.name}` is referenced by more than one model."
|
|
512
|
+
)
|