dbt-bouncer 1.31.2rc3__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_bouncer/artifact_parsers/dbt_cloud/catalog_latest.py +21 -21
- dbt_bouncer/artifact_parsers/dbt_cloud/manifest_latest.py +1745 -1745
- dbt_bouncer/artifact_parsers/dbt_cloud/run_results_latest.py +22 -22
- dbt_bouncer/artifact_parsers/parsers_catalog.py +26 -24
- dbt_bouncer/artifact_parsers/parsers_common.py +57 -36
- dbt_bouncer/artifact_parsers/parsers_manifest.py +98 -69
- dbt_bouncer/artifact_parsers/parsers_run_results.py +32 -19
- dbt_bouncer/check_base.py +22 -11
- dbt_bouncer/checks/catalog/check_catalog_sources.py +22 -12
- dbt_bouncer/checks/catalog/check_columns.py +175 -105
- dbt_bouncer/checks/common.py +24 -3
- dbt_bouncer/checks/manifest/check_exposures.py +79 -52
- dbt_bouncer/checks/manifest/check_lineage.py +69 -40
- dbt_bouncer/checks/manifest/check_macros.py +177 -104
- dbt_bouncer/checks/manifest/check_metadata.py +28 -18
- dbt_bouncer/checks/manifest/check_models.py +842 -496
- dbt_bouncer/checks/manifest/check_seeds.py +63 -0
- dbt_bouncer/checks/manifest/check_semantic_models.py +28 -20
- dbt_bouncer/checks/manifest/check_snapshots.py +57 -33
- dbt_bouncer/checks/manifest/check_sources.py +246 -137
- dbt_bouncer/checks/manifest/check_unit_tests.py +97 -54
- dbt_bouncer/checks/run_results/check_run_results.py +34 -20
- dbt_bouncer/config_file_parser.py +47 -28
- dbt_bouncer/config_file_validator.py +11 -8
- dbt_bouncer/global_context.py +31 -0
- dbt_bouncer/main.py +128 -67
- dbt_bouncer/runner.py +61 -31
- dbt_bouncer/utils.py +146 -50
- dbt_bouncer/version.py +1 -1
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/METADATA +15 -15
- dbt_bouncer-2.0.0.dist-info/RECORD +37 -0
- dbt_bouncer-1.31.2rc3.dist-info/RECORD +0 -35
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/WHEEL +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/entry_points.txt +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,25 +1,30 @@
|
|
|
1
|
-
# mypy: disable-error-code="union-attr"
|
|
2
1
|
import logging
|
|
3
2
|
import re
|
|
4
|
-
from
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import TYPE_CHECKING, Literal
|
|
5
5
|
|
|
6
6
|
from pydantic import BaseModel, ConfigDict, Field
|
|
7
7
|
|
|
8
8
|
from dbt_bouncer.check_base import BaseCheck
|
|
9
9
|
from dbt_bouncer.checks.common import NestedDict
|
|
10
|
-
from dbt_bouncer.utils import
|
|
10
|
+
from dbt_bouncer.utils import (
|
|
11
|
+
find_missing_meta_keys,
|
|
12
|
+
get_package_version_number,
|
|
13
|
+
is_description_populated,
|
|
14
|
+
)
|
|
11
15
|
|
|
12
16
|
if TYPE_CHECKING:
|
|
13
17
|
from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
|
|
14
18
|
UnitTests,
|
|
15
19
|
)
|
|
16
|
-
from dbt_bouncer.artifact_parsers.
|
|
20
|
+
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
17
21
|
DbtBouncerExposureBase,
|
|
18
22
|
DbtBouncerManifest,
|
|
19
23
|
DbtBouncerModelBase,
|
|
20
24
|
DbtBouncerTestBase,
|
|
21
25
|
)
|
|
22
26
|
|
|
27
|
+
from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
|
|
23
28
|
from dbt_bouncer.utils import clean_path_str, get_clean_model_name
|
|
24
29
|
|
|
25
30
|
|
|
@@ -33,11 +38,11 @@ class CheckModelAccess(BaseCheck):
|
|
|
33
38
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
34
39
|
|
|
35
40
|
Other Parameters:
|
|
36
|
-
description (
|
|
37
|
-
exclude (
|
|
38
|
-
include (
|
|
39
|
-
materialization (
|
|
40
|
-
severity (
|
|
41
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
42
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
43
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
44
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
45
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
41
46
|
|
|
42
47
|
Example(s):
|
|
43
48
|
```yaml
|
|
@@ -57,14 +62,22 @@ class CheckModelAccess(BaseCheck):
|
|
|
57
62
|
"""
|
|
58
63
|
|
|
59
64
|
access: Literal["private", "protected", "public"]
|
|
60
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
65
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
61
66
|
name: Literal["check_model_access"]
|
|
62
67
|
|
|
63
68
|
def execute(self) -> None:
|
|
64
|
-
"""Execute the check.
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
69
|
+
"""Execute the check.
|
|
70
|
+
|
|
71
|
+
Raises:
|
|
72
|
+
DbtBouncerFailedCheckError: If access is incorrect.
|
|
73
|
+
|
|
74
|
+
"""
|
|
75
|
+
if self.model is None:
|
|
76
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
77
|
+
if self.model.access and self.model.access.value != self.access:
|
|
78
|
+
raise DbtBouncerFailedCheckError(
|
|
79
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has `{self.model.access.value}` access, it should have access `{self.access}`."
|
|
80
|
+
)
|
|
68
81
|
|
|
69
82
|
|
|
70
83
|
class CheckModelCodeDoesNotContainRegexpPattern(BaseCheck):
|
|
@@ -77,11 +90,11 @@ class CheckModelCodeDoesNotContainRegexpPattern(BaseCheck):
|
|
|
77
90
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
78
91
|
|
|
79
92
|
Other Parameters:
|
|
80
|
-
description (
|
|
81
|
-
exclude (
|
|
82
|
-
include (
|
|
83
|
-
materialization (
|
|
84
|
-
severity (
|
|
93
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
94
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
95
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
96
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
97
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
85
98
|
|
|
86
99
|
Example(s):
|
|
87
100
|
```yaml
|
|
@@ -93,20 +106,28 @@ class CheckModelCodeDoesNotContainRegexpPattern(BaseCheck):
|
|
|
93
106
|
|
|
94
107
|
"""
|
|
95
108
|
|
|
96
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
109
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
97
110
|
name: Literal["check_model_code_does_not_contain_regexp_pattern"]
|
|
98
111
|
regexp_pattern: str
|
|
99
112
|
|
|
100
113
|
def execute(self) -> None:
|
|
101
|
-
"""Execute the check.
|
|
102
|
-
|
|
114
|
+
"""Execute the check.
|
|
115
|
+
|
|
116
|
+
Raises:
|
|
117
|
+
DbtBouncerFailedCheckError: If code contains banned string.
|
|
118
|
+
|
|
119
|
+
"""
|
|
120
|
+
if self.model is None:
|
|
121
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
122
|
+
if (
|
|
103
123
|
re.compile(self.regexp_pattern.strip(), flags=re.DOTALL).match(
|
|
104
|
-
self.model.raw_code
|
|
124
|
+
str(self.model.raw_code)
|
|
125
|
+
)
|
|
126
|
+
is not None
|
|
127
|
+
):
|
|
128
|
+
raise DbtBouncerFailedCheckError(
|
|
129
|
+
f"`{get_clean_model_name(self.model.unique_id)}` contains a banned string: `{self.regexp_pattern.strip()}`."
|
|
105
130
|
)
|
|
106
|
-
is None
|
|
107
|
-
), (
|
|
108
|
-
f"`{get_clean_model_name(self.model.unique_id)}` contains a banned string: `{self.regexp_pattern.strip()}`."
|
|
109
|
-
)
|
|
110
131
|
|
|
111
132
|
|
|
112
133
|
class CheckModelContractsEnforcedForPublicModel(BaseCheck):
|
|
@@ -116,11 +137,11 @@ class CheckModelContractsEnforcedForPublicModel(BaseCheck):
|
|
|
116
137
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
117
138
|
|
|
118
139
|
Other Parameters:
|
|
119
|
-
description (
|
|
120
|
-
exclude (
|
|
121
|
-
include (
|
|
122
|
-
materialization (
|
|
123
|
-
severity (
|
|
140
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
141
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
142
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
143
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
144
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
124
145
|
|
|
125
146
|
Example(s):
|
|
126
147
|
```yaml
|
|
@@ -130,13 +151,24 @@ class CheckModelContractsEnforcedForPublicModel(BaseCheck):
|
|
|
130
151
|
|
|
131
152
|
"""
|
|
132
153
|
|
|
133
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
154
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
134
155
|
name: Literal["check_model_contract_enforced_for_public_model"]
|
|
135
156
|
|
|
136
157
|
def execute(self) -> None:
|
|
137
|
-
"""Execute the check.
|
|
138
|
-
|
|
139
|
-
|
|
158
|
+
"""Execute the check.
|
|
159
|
+
|
|
160
|
+
Raises:
|
|
161
|
+
DbtBouncerFailedCheckError: If contracts are not enforced for public model.
|
|
162
|
+
|
|
163
|
+
"""
|
|
164
|
+
if self.model is None:
|
|
165
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
166
|
+
if (
|
|
167
|
+
self.model.access
|
|
168
|
+
and self.model.access.value == "public"
|
|
169
|
+
and (not self.model.contract or self.model.contract.enforced is not True)
|
|
170
|
+
):
|
|
171
|
+
raise DbtBouncerFailedCheckError(
|
|
140
172
|
f"`{get_clean_model_name(self.model.unique_id)}` is a public model but does not have contracts enforced."
|
|
141
173
|
)
|
|
142
174
|
|
|
@@ -145,18 +177,18 @@ class CheckModelDependsOnMacros(BaseCheck):
|
|
|
145
177
|
"""Models must depend on the specified macros.
|
|
146
178
|
|
|
147
179
|
Parameters:
|
|
148
|
-
criteria: (
|
|
149
|
-
required_macros: (
|
|
180
|
+
criteria: (Literal["any", "all", "one"] | None): Whether the model must depend on any, all, or exactly one of the specified macros. Default: `any`.
|
|
181
|
+
required_macros: (list[str]): List of macros the model must depend on. All macros must specify a namespace, e.g. `dbt.is_incremental`.
|
|
150
182
|
|
|
151
183
|
Receives:
|
|
152
184
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
153
185
|
|
|
154
186
|
Other Parameters:
|
|
155
|
-
description (
|
|
156
|
-
exclude (
|
|
157
|
-
include (
|
|
158
|
-
materialization (
|
|
159
|
-
severity (
|
|
187
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
188
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
189
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
190
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
191
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
160
192
|
|
|
161
193
|
Example(s):
|
|
162
194
|
```yaml
|
|
@@ -174,30 +206,41 @@ class CheckModelDependsOnMacros(BaseCheck):
|
|
|
174
206
|
"""
|
|
175
207
|
|
|
176
208
|
criteria: Literal["any", "all", "one"] = Field(default="all")
|
|
177
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
209
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
178
210
|
name: Literal["check_model_depends_on_macros"]
|
|
179
|
-
required_macros:
|
|
211
|
+
required_macros: list[str]
|
|
180
212
|
|
|
181
213
|
def execute(self) -> None:
|
|
182
|
-
"""Execute the check.
|
|
214
|
+
"""Execute the check.
|
|
215
|
+
|
|
216
|
+
Raises:
|
|
217
|
+
DbtBouncerFailedCheckError: If model does not depend on required macros.
|
|
218
|
+
|
|
219
|
+
"""
|
|
220
|
+
if self.model is None:
|
|
221
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
183
222
|
upstream_macros = [
|
|
184
|
-
(".").join(m.split(".")[1:])
|
|
223
|
+
(".").join(m.split(".")[1:])
|
|
224
|
+
for m in getattr(self.model.depends_on, "macros", []) or []
|
|
185
225
|
]
|
|
186
226
|
if self.criteria == "any":
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
227
|
+
if not any(macro in upstream_macros for macro in self.required_macros):
|
|
228
|
+
raise DbtBouncerFailedCheckError(
|
|
229
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not depend on any of the required macros: {self.required_macros}."
|
|
230
|
+
)
|
|
190
231
|
elif self.criteria == "all":
|
|
191
232
|
missing_macros = [
|
|
192
233
|
macro for macro in self.required_macros if macro not in upstream_macros
|
|
193
234
|
]
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
)
|
|
235
|
+
if missing_macros:
|
|
236
|
+
raise DbtBouncerFailedCheckError(
|
|
237
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is missing required macros: {missing_macros}."
|
|
238
|
+
)
|
|
239
|
+
elif (
|
|
240
|
+
self.criteria == "one"
|
|
241
|
+
and sum(macro in upstream_macros for macro in self.required_macros) != 1
|
|
242
|
+
):
|
|
243
|
+
raise DbtBouncerFailedCheckError(
|
|
201
244
|
f"`{get_clean_model_name(self.model.unique_id)}` must depend on exactly one of the required macros: {self.required_macros}."
|
|
202
245
|
)
|
|
203
246
|
|
|
@@ -209,11 +252,11 @@ class CheckModelDependsOnMultipleSources(BaseCheck):
|
|
|
209
252
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
210
253
|
|
|
211
254
|
Other Parameters:
|
|
212
|
-
description (
|
|
213
|
-
exclude (
|
|
214
|
-
include (
|
|
215
|
-
materialization (
|
|
216
|
-
severity (
|
|
255
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
256
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
257
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
258
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
259
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
217
260
|
|
|
218
261
|
Example(s):
|
|
219
262
|
```yaml
|
|
@@ -223,17 +266,26 @@ class CheckModelDependsOnMultipleSources(BaseCheck):
|
|
|
223
266
|
|
|
224
267
|
"""
|
|
225
268
|
|
|
226
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
269
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
227
270
|
name: Literal["check_model_depends_on_multiple_sources"]
|
|
228
271
|
|
|
229
272
|
def execute(self) -> None:
|
|
230
|
-
"""Execute the check.
|
|
273
|
+
"""Execute the check.
|
|
274
|
+
|
|
275
|
+
Raises:
|
|
276
|
+
DbtBouncerFailedCheckError: If model references more than one source.
|
|
277
|
+
|
|
278
|
+
"""
|
|
279
|
+
if self.model is None:
|
|
280
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
231
281
|
num_reffed_sources = sum(
|
|
232
|
-
x.split(".")[0] == "source"
|
|
233
|
-
|
|
234
|
-
assert num_reffed_sources <= 1, (
|
|
235
|
-
f"`{get_clean_model_name(self.model.unique_id)}` references more than one source."
|
|
282
|
+
x.split(".")[0] == "source"
|
|
283
|
+
for x in getattr(self.model.depends_on, "nodes", []) or []
|
|
236
284
|
)
|
|
285
|
+
if num_reffed_sources > 1:
|
|
286
|
+
raise DbtBouncerFailedCheckError(
|
|
287
|
+
f"`{get_clean_model_name(self.model.unique_id)}` references more than one source."
|
|
288
|
+
)
|
|
237
289
|
|
|
238
290
|
|
|
239
291
|
class CheckModelDescriptionContainsRegexPattern(BaseCheck):
|
|
@@ -244,11 +296,11 @@ class CheckModelDescriptionContainsRegexPattern(BaseCheck):
|
|
|
244
296
|
regexp_pattern (str): The regexp pattern that should match the model description.
|
|
245
297
|
|
|
246
298
|
Other Parameters:
|
|
247
|
-
description (
|
|
248
|
-
exclude (
|
|
249
|
-
include (
|
|
250
|
-
materialization (
|
|
251
|
-
severity (
|
|
299
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
300
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
301
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
302
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
303
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
252
304
|
|
|
253
305
|
Example(s):
|
|
254
306
|
```yaml
|
|
@@ -259,48 +311,75 @@ class CheckModelDescriptionContainsRegexPattern(BaseCheck):
|
|
|
259
311
|
|
|
260
312
|
"""
|
|
261
313
|
|
|
262
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
314
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
263
315
|
name: Literal["check_model_description_contains_regex_pattern"]
|
|
264
316
|
regexp_pattern: str
|
|
265
317
|
|
|
266
318
|
def execute(self) -> None:
|
|
267
|
-
"""Execute the check.
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
319
|
+
"""Execute the check.
|
|
320
|
+
|
|
321
|
+
Raises:
|
|
322
|
+
DbtBouncerFailedCheckError: If description does not match regex.
|
|
323
|
+
|
|
324
|
+
"""
|
|
325
|
+
if self.model is None:
|
|
326
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
327
|
+
if not re.compile(self.regexp_pattern.strip(), flags=re.DOTALL).match(
|
|
328
|
+
str(self.model.description)
|
|
329
|
+
):
|
|
330
|
+
raise DbtBouncerFailedCheckError(
|
|
331
|
+
f"""`{get_clean_model_name(self.model.unique_id)}`'s description "{self.model.description}" doesn't match the supplied regex: {self.regexp_pattern}."""
|
|
332
|
+
)
|
|
273
333
|
|
|
274
334
|
|
|
275
335
|
class CheckModelDescriptionPopulated(BaseCheck):
|
|
276
336
|
"""Models must have a populated description.
|
|
277
337
|
|
|
338
|
+
Parameters:
|
|
339
|
+
min_description_length (int | None): Minimum length required for the description to be considered populated.
|
|
340
|
+
|
|
278
341
|
Receives:
|
|
279
342
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
280
343
|
|
|
281
344
|
Other Parameters:
|
|
282
|
-
description (
|
|
283
|
-
exclude (
|
|
284
|
-
include (
|
|
285
|
-
materialization (
|
|
286
|
-
severity (
|
|
345
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
346
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
347
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
348
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
349
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
287
350
|
|
|
288
351
|
Example(s):
|
|
289
352
|
```yaml
|
|
290
353
|
manifest_checks:
|
|
291
354
|
- name: check_model_description_populated
|
|
292
355
|
```
|
|
356
|
+
```yaml
|
|
357
|
+
manifest_checks:
|
|
358
|
+
- name: check_model_description_populated
|
|
359
|
+
min_description_length: 25 # Setting a stricter requirement for description length
|
|
360
|
+
```
|
|
293
361
|
|
|
294
362
|
"""
|
|
295
363
|
|
|
296
|
-
|
|
364
|
+
min_description_length: int | None = Field(default=None)
|
|
365
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
297
366
|
name: Literal["check_model_description_populated"]
|
|
298
367
|
|
|
299
368
|
def execute(self) -> None:
|
|
300
|
-
"""Execute the check.
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
369
|
+
"""Execute the check.
|
|
370
|
+
|
|
371
|
+
Raises:
|
|
372
|
+
DbtBouncerFailedCheckError: If description is not populated.
|
|
373
|
+
|
|
374
|
+
"""
|
|
375
|
+
if self.model is None:
|
|
376
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
377
|
+
if not self._is_description_populated(
|
|
378
|
+
self.model.description or "", self.min_description_length
|
|
379
|
+
):
|
|
380
|
+
raise DbtBouncerFailedCheckError(
|
|
381
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have a populated description."
|
|
382
|
+
)
|
|
304
383
|
|
|
305
384
|
|
|
306
385
|
class CheckModelDirectories(BaseCheck):
|
|
@@ -308,17 +387,17 @@ class CheckModelDirectories(BaseCheck):
|
|
|
308
387
|
|
|
309
388
|
Parameters:
|
|
310
389
|
include (str): Regex pattern to the directory to check.
|
|
311
|
-
permitted_sub_directories (
|
|
390
|
+
permitted_sub_directories (list[str]): List of permitted sub-directories.
|
|
312
391
|
|
|
313
392
|
Receives:
|
|
314
393
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
315
394
|
|
|
316
395
|
Other Parameters:
|
|
317
|
-
description (
|
|
318
|
-
exclude (
|
|
319
|
-
include (
|
|
320
|
-
materialization (
|
|
321
|
-
severity (
|
|
396
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
397
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
398
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
399
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
400
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
322
401
|
|
|
323
402
|
Example(s):
|
|
324
403
|
```yaml
|
|
@@ -342,33 +421,35 @@ class CheckModelDirectories(BaseCheck):
|
|
|
342
421
|
"""
|
|
343
422
|
|
|
344
423
|
include: str
|
|
345
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
424
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
346
425
|
name: Literal["check_model_directories"]
|
|
347
|
-
permitted_sub_directories:
|
|
426
|
+
permitted_sub_directories: list[str]
|
|
348
427
|
|
|
349
428
|
def execute(self) -> None:
|
|
350
429
|
"""Execute the check.
|
|
351
430
|
|
|
352
431
|
Raises:
|
|
353
|
-
|
|
432
|
+
DbtBouncerFailedCheckError: If model located in `./models` or invalid subdirectory.
|
|
354
433
|
|
|
355
434
|
"""
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
)
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
435
|
+
if self.model is None:
|
|
436
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
437
|
+
clean_path = clean_path_str(self.model.original_file_path)
|
|
438
|
+
matched_path = re.compile(self.include.strip().rstrip("/")).match(clean_path)
|
|
439
|
+
if matched_path is None:
|
|
440
|
+
raise DbtBouncerFailedCheckError("matched_path is None")
|
|
441
|
+
path_after_match = clean_path[matched_path.end() + 1 :]
|
|
442
|
+
directory_to_check = Path(path_after_match).parts[0]
|
|
363
443
|
|
|
364
444
|
if directory_to_check.replace(".sql", "") == self.model.name:
|
|
365
|
-
raise
|
|
445
|
+
raise DbtBouncerFailedCheckError(
|
|
366
446
|
f"`{get_clean_model_name(self.model.unique_id)}` is not located in a valid sub-directory ({self.permitted_sub_directories})."
|
|
367
447
|
)
|
|
368
448
|
else:
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
449
|
+
if directory_to_check not in self.permitted_sub_directories:
|
|
450
|
+
raise DbtBouncerFailedCheckError(
|
|
451
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is located in the `{directory_to_check}` sub-directory, this is not a valid sub-directory ({self.permitted_sub_directories})."
|
|
452
|
+
)
|
|
372
453
|
|
|
373
454
|
|
|
374
455
|
class CheckModelDocumentedInSameDirectory(BaseCheck):
|
|
@@ -378,11 +459,11 @@ class CheckModelDocumentedInSameDirectory(BaseCheck):
|
|
|
378
459
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
379
460
|
|
|
380
461
|
Other Parameters:
|
|
381
|
-
description (
|
|
382
|
-
exclude (
|
|
383
|
-
include (
|
|
384
|
-
materialization (
|
|
385
|
-
severity (
|
|
462
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
463
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
464
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
465
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
466
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
386
467
|
|
|
387
468
|
Example(s):
|
|
388
469
|
```yaml
|
|
@@ -392,26 +473,41 @@ class CheckModelDocumentedInSameDirectory(BaseCheck):
|
|
|
392
473
|
|
|
393
474
|
"""
|
|
394
475
|
|
|
395
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
476
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
396
477
|
name: Literal["check_model_documented_in_same_directory"]
|
|
397
478
|
|
|
398
479
|
def execute(self) -> None:
|
|
399
|
-
"""Execute the check.
|
|
400
|
-
|
|
401
|
-
|
|
480
|
+
"""Execute the check.
|
|
481
|
+
|
|
482
|
+
Raises:
|
|
483
|
+
DbtBouncerFailedCheckError: If model is not documented in same directory.
|
|
484
|
+
|
|
485
|
+
"""
|
|
486
|
+
if self.model is None:
|
|
487
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
488
|
+
model_sql_path = Path(clean_path_str(self.model.original_file_path))
|
|
489
|
+
model_sql_dir = model_sql_path.parent.parts
|
|
490
|
+
|
|
491
|
+
if not (
|
|
402
492
|
hasattr(self.model, "patch_path")
|
|
403
|
-
and clean_path_str(self.model.patch_path) is not None
|
|
404
|
-
)
|
|
493
|
+
and clean_path_str(self.model.patch_path or "") is not None
|
|
494
|
+
):
|
|
495
|
+
raise DbtBouncerFailedCheckError(
|
|
496
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is not documented."
|
|
497
|
+
)
|
|
405
498
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
]
|
|
410
|
-
).split("/")[:-1]
|
|
499
|
+
patch_path_str = clean_path_str(self.model.patch_path or "")
|
|
500
|
+
start_idx = patch_path_str.find("models")
|
|
501
|
+
if start_idx != -1:
|
|
502
|
+
patch_path_str = patch_path_str[start_idx:]
|
|
411
503
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
504
|
+
model_doc_path = Path(patch_path_str)
|
|
505
|
+
model_doc_dir = model_doc_path.parent.parts
|
|
506
|
+
|
|
507
|
+
if model_doc_dir != model_sql_dir:
|
|
508
|
+
raise DbtBouncerFailedCheckError(
|
|
509
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is documented in a different directory to the `.sql` file: `{'/'.join(model_doc_dir)}` vs `{'/'.join(model_sql_dir)}`."
|
|
510
|
+
)
|
|
415
511
|
|
|
416
512
|
|
|
417
513
|
class CheckModelFileName(BaseCheck):
|
|
@@ -424,11 +520,11 @@ class CheckModelFileName(BaseCheck):
|
|
|
424
520
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
425
521
|
|
|
426
522
|
Other Parameters:
|
|
427
|
-
description (
|
|
428
|
-
exclude (
|
|
429
|
-
include (
|
|
430
|
-
materialization (
|
|
431
|
-
severity (
|
|
523
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
524
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
525
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
526
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
527
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
432
528
|
|
|
433
529
|
Example(s):
|
|
434
530
|
```yaml
|
|
@@ -442,17 +538,23 @@ class CheckModelFileName(BaseCheck):
|
|
|
442
538
|
"""
|
|
443
539
|
|
|
444
540
|
file_name_pattern: str
|
|
445
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
541
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
446
542
|
name: Literal["check_model_file_name"]
|
|
447
543
|
|
|
448
544
|
def execute(self) -> None:
|
|
449
|
-
"""Execute the check.
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
545
|
+
"""Execute the check.
|
|
546
|
+
|
|
547
|
+
Raises:
|
|
548
|
+
DbtBouncerFailedCheckError: If file name does not match regex.
|
|
549
|
+
|
|
550
|
+
"""
|
|
551
|
+
if self.model is None:
|
|
552
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
553
|
+
file_name = Path(clean_path_str(self.model.original_file_path)).name
|
|
554
|
+
if re.compile(self.file_name_pattern.strip()).match(file_name) is None:
|
|
555
|
+
raise DbtBouncerFailedCheckError(
|
|
556
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is in a file that does not match the supplied regex `{self.file_name_pattern.strip()}`."
|
|
557
|
+
)
|
|
456
558
|
|
|
457
559
|
|
|
458
560
|
class CheckModelGrantPrivilege(BaseCheck):
|
|
@@ -463,11 +565,11 @@ class CheckModelGrantPrivilege(BaseCheck):
|
|
|
463
565
|
privilege_pattern (str): Regex pattern to match the privilege.
|
|
464
566
|
|
|
465
567
|
Other Parameters:
|
|
466
|
-
description (
|
|
467
|
-
exclude (
|
|
468
|
-
include (
|
|
469
|
-
materialization (
|
|
470
|
-
severity (
|
|
568
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
569
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
570
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
571
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
572
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
471
573
|
|
|
472
574
|
Example(s):
|
|
473
575
|
```yaml
|
|
@@ -479,21 +581,31 @@ class CheckModelGrantPrivilege(BaseCheck):
|
|
|
479
581
|
|
|
480
582
|
"""
|
|
481
583
|
|
|
482
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
584
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
483
585
|
name: Literal["check_model_grant_privilege"]
|
|
484
586
|
privilege_pattern: str
|
|
485
587
|
|
|
486
588
|
def execute(self) -> None:
|
|
487
|
-
"""Execute the check.
|
|
589
|
+
"""Execute the check.
|
|
590
|
+
|
|
591
|
+
Raises:
|
|
592
|
+
DbtBouncerFailedCheckError: If grant privileges do not match regex.
|
|
593
|
+
|
|
594
|
+
"""
|
|
595
|
+
if self.model is None:
|
|
596
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
597
|
+
config = self.model.config
|
|
598
|
+
grants = config.grants if config else {}
|
|
488
599
|
non_complying_grants = [
|
|
489
600
|
i
|
|
490
|
-
for i in
|
|
491
|
-
if re.compile(self.privilege_pattern.strip()).match(i) is None
|
|
601
|
+
for i in (grants or {})
|
|
602
|
+
if re.compile(self.privilege_pattern.strip()).match(str(i)) is None
|
|
492
603
|
]
|
|
493
604
|
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
605
|
+
if non_complying_grants:
|
|
606
|
+
raise DbtBouncerFailedCheckError(
|
|
607
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has grants (`{self.privilege_pattern}`) that don't comply with the specified regexp pattern ({non_complying_grants})."
|
|
608
|
+
)
|
|
497
609
|
|
|
498
610
|
|
|
499
611
|
class CheckModelGrantPrivilegeRequired(BaseCheck):
|
|
@@ -504,11 +616,11 @@ class CheckModelGrantPrivilegeRequired(BaseCheck):
|
|
|
504
616
|
privilege (str): The privilege that is required.
|
|
505
617
|
|
|
506
618
|
Other Parameters:
|
|
507
|
-
description (
|
|
508
|
-
exclude (
|
|
509
|
-
include (
|
|
510
|
-
materialization (
|
|
511
|
-
severity (
|
|
619
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
620
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
621
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
622
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
623
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
512
624
|
|
|
513
625
|
Example(s):
|
|
514
626
|
```yaml
|
|
@@ -520,15 +632,25 @@ class CheckModelGrantPrivilegeRequired(BaseCheck):
|
|
|
520
632
|
|
|
521
633
|
"""
|
|
522
634
|
|
|
523
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
635
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
524
636
|
name: Literal["check_model_grant_privilege_required"]
|
|
525
637
|
privilege: str
|
|
526
638
|
|
|
527
639
|
def execute(self) -> None:
|
|
528
|
-
"""Execute the check.
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
640
|
+
"""Execute the check.
|
|
641
|
+
|
|
642
|
+
Raises:
|
|
643
|
+
DbtBouncerFailedCheckError: If required grant privilege is missing.
|
|
644
|
+
|
|
645
|
+
"""
|
|
646
|
+
if self.model is None:
|
|
647
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
648
|
+
config = self.model.config
|
|
649
|
+
grants = config.grants if config else {}
|
|
650
|
+
if self.privilege not in (grants or {}):
|
|
651
|
+
raise DbtBouncerFailedCheckError(
|
|
652
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have the required grant privilege (`{self.privilege}`)."
|
|
653
|
+
)
|
|
532
654
|
|
|
533
655
|
|
|
534
656
|
class CheckModelHasContractsEnforced(BaseCheck):
|
|
@@ -538,11 +660,11 @@ class CheckModelHasContractsEnforced(BaseCheck):
|
|
|
538
660
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
539
661
|
|
|
540
662
|
Other Parameters:
|
|
541
|
-
description (
|
|
542
|
-
exclude (
|
|
543
|
-
include (
|
|
544
|
-
materialization (
|
|
545
|
-
severity (
|
|
663
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
664
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
665
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
666
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
667
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
546
668
|
|
|
547
669
|
Example(s):
|
|
548
670
|
```yaml
|
|
@@ -553,29 +675,37 @@ class CheckModelHasContractsEnforced(BaseCheck):
|
|
|
553
675
|
|
|
554
676
|
"""
|
|
555
677
|
|
|
556
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
678
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
557
679
|
name: Literal["check_model_has_contracts_enforced"]
|
|
558
680
|
|
|
559
681
|
def execute(self) -> None:
|
|
560
|
-
"""Execute the check.
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
682
|
+
"""Execute the check.
|
|
683
|
+
|
|
684
|
+
Raises:
|
|
685
|
+
DbtBouncerFailedCheckError: If contracts are not enforced.
|
|
686
|
+
|
|
687
|
+
"""
|
|
688
|
+
if self.model is None:
|
|
689
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
690
|
+
if not self.model.contract or self.model.contract.enforced is not True:
|
|
691
|
+
raise DbtBouncerFailedCheckError(
|
|
692
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have contracts enforced."
|
|
693
|
+
)
|
|
564
694
|
|
|
565
695
|
|
|
566
696
|
class CheckModelHasExposure(BaseCheck):
|
|
567
697
|
"""Models must have an exposure.
|
|
568
698
|
|
|
569
699
|
Receives:
|
|
570
|
-
exposures (
|
|
700
|
+
exposures (list[DbtBouncerExposureBase]): List of DbtBouncerExposureBase objects parsed from `manifest.json`.
|
|
571
701
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
572
702
|
|
|
573
703
|
Other Parameters:
|
|
574
|
-
description (
|
|
575
|
-
exclude (
|
|
576
|
-
include (
|
|
577
|
-
materialization (
|
|
578
|
-
severity (
|
|
704
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
705
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
706
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
707
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
708
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
579
709
|
|
|
580
710
|
Example(s):
|
|
581
711
|
```yaml
|
|
@@ -589,21 +719,29 @@ class CheckModelHasExposure(BaseCheck):
|
|
|
589
719
|
|
|
590
720
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
591
721
|
|
|
592
|
-
exposures:
|
|
593
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
722
|
+
exposures: list["DbtBouncerExposureBase"] = Field(default=[])
|
|
723
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
594
724
|
name: Literal["check_model_has_exposure"]
|
|
595
725
|
|
|
596
726
|
def execute(self) -> None:
|
|
597
|
-
"""Execute the check.
|
|
727
|
+
"""Execute the check.
|
|
728
|
+
|
|
729
|
+
Raises:
|
|
730
|
+
DbtBouncerFailedCheckError: If model does not have an exposure.
|
|
731
|
+
|
|
732
|
+
"""
|
|
733
|
+
if self.model is None:
|
|
734
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
598
735
|
has_exposure = False
|
|
599
736
|
for e in self.exposures:
|
|
600
|
-
for m in e.depends_on
|
|
737
|
+
for m in getattr(e.depends_on, "nodes", []) or []:
|
|
601
738
|
if m == self.model.unique_id:
|
|
602
739
|
has_exposure = True
|
|
603
740
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
741
|
+
if not has_exposure:
|
|
742
|
+
raise DbtBouncerFailedCheckError(
|
|
743
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have an associated exposure."
|
|
744
|
+
)
|
|
607
745
|
|
|
608
746
|
|
|
609
747
|
class CheckModelHasMetaKeys(BaseCheck):
|
|
@@ -614,11 +752,11 @@ class CheckModelHasMetaKeys(BaseCheck):
|
|
|
614
752
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
615
753
|
|
|
616
754
|
Other Parameters:
|
|
617
|
-
description (
|
|
618
|
-
exclude (
|
|
619
|
-
include (
|
|
620
|
-
materialization (
|
|
621
|
-
severity (
|
|
755
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
756
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
757
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
758
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
759
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
622
760
|
|
|
623
761
|
Example(s):
|
|
624
762
|
```yaml
|
|
@@ -632,18 +770,26 @@ class CheckModelHasMetaKeys(BaseCheck):
|
|
|
632
770
|
"""
|
|
633
771
|
|
|
634
772
|
keys: NestedDict
|
|
635
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
773
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
636
774
|
name: Literal["check_model_has_meta_keys"]
|
|
637
775
|
|
|
638
776
|
def execute(self) -> None:
|
|
639
|
-
"""Execute the check.
|
|
777
|
+
"""Execute the check.
|
|
778
|
+
|
|
779
|
+
Raises:
|
|
780
|
+
DbtBouncerFailedCheckError: If required meta keys are missing.
|
|
781
|
+
|
|
782
|
+
"""
|
|
783
|
+
if self.model is None:
|
|
784
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
640
785
|
missing_keys = find_missing_meta_keys(
|
|
641
786
|
meta_config=self.model.meta,
|
|
642
787
|
required_keys=self.keys.model_dump(),
|
|
643
788
|
)
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
789
|
+
if missing_keys != []:
|
|
790
|
+
raise DbtBouncerFailedCheckError(
|
|
791
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is missing the following keys from the `meta` config: {[x.replace('>>', '') for x in missing_keys]}"
|
|
792
|
+
)
|
|
647
793
|
|
|
648
794
|
|
|
649
795
|
class CheckModelHasNoUpstreamDependencies(BaseCheck):
|
|
@@ -653,11 +799,11 @@ class CheckModelHasNoUpstreamDependencies(BaseCheck):
|
|
|
653
799
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
654
800
|
|
|
655
801
|
Other Parameters:
|
|
656
|
-
description (
|
|
657
|
-
exclude (
|
|
658
|
-
include (
|
|
659
|
-
materialization (
|
|
660
|
-
severity (
|
|
802
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
803
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
804
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
805
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
806
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
661
807
|
|
|
662
808
|
Example(s):
|
|
663
809
|
```yaml
|
|
@@ -667,14 +813,26 @@ class CheckModelHasNoUpstreamDependencies(BaseCheck):
|
|
|
667
813
|
|
|
668
814
|
"""
|
|
669
815
|
|
|
670
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
816
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
671
817
|
name: Literal["check_model_has_no_upstream_dependencies"]
|
|
672
818
|
|
|
673
819
|
def execute(self) -> None:
|
|
674
|
-
"""Execute the check.
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
820
|
+
"""Execute the check.
|
|
821
|
+
|
|
822
|
+
Raises:
|
|
823
|
+
DbtBouncerFailedCheckError: If model has no upstream dependencies.
|
|
824
|
+
|
|
825
|
+
"""
|
|
826
|
+
if self.model is None:
|
|
827
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
828
|
+
if (
|
|
829
|
+
not self.model.depends_on
|
|
830
|
+
or not self.model.depends_on.nodes
|
|
831
|
+
or len(self.model.depends_on.nodes) <= 0
|
|
832
|
+
):
|
|
833
|
+
raise DbtBouncerFailedCheckError(
|
|
834
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has no upstream dependencies, this likely indicates hard-coded tables references."
|
|
835
|
+
)
|
|
678
836
|
|
|
679
837
|
|
|
680
838
|
class CheckModelHasSemiColon(BaseCheck):
|
|
@@ -684,11 +842,11 @@ class CheckModelHasSemiColon(BaseCheck):
|
|
|
684
842
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
685
843
|
|
|
686
844
|
Other Parameters:
|
|
687
|
-
description (
|
|
688
|
-
exclude (
|
|
689
|
-
include (
|
|
690
|
-
materialization (
|
|
691
|
-
severity (
|
|
845
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
846
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
847
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
848
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
849
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
692
850
|
|
|
693
851
|
Example(s):
|
|
694
852
|
```yaml
|
|
@@ -699,30 +857,38 @@ class CheckModelHasSemiColon(BaseCheck):
|
|
|
699
857
|
|
|
700
858
|
"""
|
|
701
859
|
|
|
702
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
860
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
703
861
|
name: Literal["check_model_has_semi_colon"]
|
|
704
862
|
|
|
705
863
|
def execute(self) -> None:
|
|
706
|
-
"""Execute the check.
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
864
|
+
"""Execute the check.
|
|
865
|
+
|
|
866
|
+
Raises:
|
|
867
|
+
DbtBouncerFailedCheckError: If model ends with a semi-colon.
|
|
868
|
+
|
|
869
|
+
"""
|
|
870
|
+
if self.model is None:
|
|
871
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
872
|
+
if (self.model.raw_code or "").strip()[-1] == ";":
|
|
873
|
+
raise DbtBouncerFailedCheckError(
|
|
874
|
+
f"`{get_clean_model_name(self.model.unique_id)}` ends with a semi-colon, this is not permitted."
|
|
875
|
+
)
|
|
710
876
|
|
|
711
877
|
|
|
712
878
|
class CheckModelHasTags(BaseCheck):
|
|
713
879
|
"""Models must have the specified tags.
|
|
714
880
|
|
|
715
881
|
Parameters:
|
|
716
|
-
criteria: (
|
|
882
|
+
criteria: (Literal["any", "all", "one"] | None): Whether the model must have any, all, or exactly one of the specified tags. Default: `any`.
|
|
717
883
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
718
|
-
tags (
|
|
884
|
+
tags (list[str]): List of tags to check for.
|
|
719
885
|
|
|
720
886
|
Other Parameters:
|
|
721
|
-
description (
|
|
722
|
-
exclude (
|
|
723
|
-
include (
|
|
724
|
-
materialization (
|
|
725
|
-
severity (
|
|
887
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
888
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
889
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
890
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
891
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
726
892
|
|
|
727
893
|
Example(s):
|
|
728
894
|
```yaml
|
|
@@ -736,23 +902,35 @@ class CheckModelHasTags(BaseCheck):
|
|
|
736
902
|
"""
|
|
737
903
|
|
|
738
904
|
criteria: Literal["any", "all", "one"] = Field(default="all")
|
|
739
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
905
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
740
906
|
name: Literal["check_model_has_tags"]
|
|
741
|
-
tags:
|
|
907
|
+
tags: list[str]
|
|
742
908
|
|
|
743
909
|
def execute(self) -> None:
|
|
744
|
-
"""Execute the check.
|
|
910
|
+
"""Execute the check.
|
|
911
|
+
|
|
912
|
+
Raises:
|
|
913
|
+
DbtBouncerFailedCheckError: If model does not have required tags.
|
|
914
|
+
|
|
915
|
+
"""
|
|
916
|
+
if self.model is None:
|
|
917
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
918
|
+
model_tags = self.model.tags or []
|
|
745
919
|
if self.criteria == "any":
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
920
|
+
if not any(tag in model_tags for tag in self.tags):
|
|
921
|
+
raise DbtBouncerFailedCheckError(
|
|
922
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have any of the required tags: {self.tags}."
|
|
923
|
+
)
|
|
749
924
|
elif self.criteria == "all":
|
|
750
|
-
missing_tags = [tag for tag in self.tags if tag not in
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
925
|
+
missing_tags = [tag for tag in self.tags if tag not in model_tags]
|
|
926
|
+
if missing_tags:
|
|
927
|
+
raise DbtBouncerFailedCheckError(
|
|
928
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is missing required tags: {missing_tags}."
|
|
929
|
+
)
|
|
930
|
+
elif (
|
|
931
|
+
self.criteria == "one" and sum(tag in model_tags for tag in self.tags) != 1
|
|
932
|
+
):
|
|
933
|
+
raise DbtBouncerFailedCheckError(
|
|
756
934
|
f"`{get_clean_model_name(self.model.unique_id)}` must have exactly one of the required tags: {self.tags}."
|
|
757
935
|
)
|
|
758
936
|
|
|
@@ -761,16 +939,16 @@ class CheckModelHasUniqueTest(BaseCheck):
|
|
|
761
939
|
"""Models must have a test for uniqueness of a column.
|
|
762
940
|
|
|
763
941
|
Parameters:
|
|
764
|
-
accepted_uniqueness_tests (
|
|
942
|
+
accepted_uniqueness_tests (list[str] | None): List of tests that are accepted as uniqueness tests.
|
|
765
943
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
766
|
-
tests (
|
|
944
|
+
tests (list[DbtBouncerTestBase]): List of DbtBouncerTestBase objects parsed from `manifest.json`.
|
|
767
945
|
|
|
768
946
|
Other Parameters:
|
|
769
|
-
description (
|
|
770
|
-
exclude (
|
|
771
|
-
include (
|
|
772
|
-
materialization (
|
|
773
|
-
severity (
|
|
947
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
948
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
949
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
950
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
951
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
774
952
|
|
|
775
953
|
Example(s):
|
|
776
954
|
```yaml
|
|
@@ -790,56 +968,69 @@ class CheckModelHasUniqueTest(BaseCheck):
|
|
|
790
968
|
|
|
791
969
|
"""
|
|
792
970
|
|
|
793
|
-
accepted_uniqueness_tests:
|
|
971
|
+
accepted_uniqueness_tests: list[str] | None = Field(
|
|
794
972
|
default=[
|
|
795
973
|
"dbt_expectations.expect_compound_columns_to_be_unique",
|
|
796
974
|
"dbt_utils.unique_combination_of_columns",
|
|
797
975
|
"unique",
|
|
798
976
|
],
|
|
799
977
|
)
|
|
800
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
978
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
801
979
|
name: Literal["check_model_has_unique_test"]
|
|
802
|
-
tests:
|
|
980
|
+
tests: list["DbtBouncerTestBase"] = Field(default=[])
|
|
803
981
|
|
|
804
982
|
def execute(self) -> None:
|
|
805
|
-
"""Execute the check.
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
983
|
+
"""Execute the check.
|
|
984
|
+
|
|
985
|
+
Raises:
|
|
986
|
+
DbtBouncerFailedCheckError: If model does not have a unique test.
|
|
987
|
+
|
|
988
|
+
"""
|
|
989
|
+
if self.model is None:
|
|
990
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
991
|
+
num_unique_tests = 0
|
|
992
|
+
for test in self.tests:
|
|
993
|
+
test_metadata = getattr(test, "test_metadata", None)
|
|
994
|
+
attached_node = getattr(test, "attached_node", None)
|
|
995
|
+
if (
|
|
996
|
+
test_metadata
|
|
997
|
+
and attached_node == self.model.unique_id
|
|
998
|
+
and (
|
|
999
|
+
(
|
|
1000
|
+
f"{getattr(test_metadata, 'namespace', '')}.{getattr(test_metadata, 'name', '')}"
|
|
1001
|
+
in (self.accepted_uniqueness_tests or [])
|
|
1002
|
+
)
|
|
1003
|
+
or (
|
|
1004
|
+
getattr(test_metadata, "namespace", None) is None
|
|
1005
|
+
and getattr(test_metadata, "name", "")
|
|
1006
|
+
in (self.accepted_uniqueness_tests or [])
|
|
1007
|
+
)
|
|
1008
|
+
)
|
|
1009
|
+
):
|
|
1010
|
+
num_unique_tests += 1
|
|
1011
|
+
if num_unique_tests < 1:
|
|
1012
|
+
raise DbtBouncerFailedCheckError(
|
|
1013
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not have a test for uniqueness of a column."
|
|
817
1014
|
)
|
|
818
|
-
for test in self.tests
|
|
819
|
-
if hasattr(test, "test_metadata")
|
|
820
|
-
)
|
|
821
|
-
assert num_unique_tests >= 1, (
|
|
822
|
-
f"`{get_clean_model_name(self.model.unique_id)}` does not have a test for uniqueness of a column."
|
|
823
|
-
)
|
|
824
1015
|
|
|
825
1016
|
|
|
826
1017
|
class CheckModelHasUnitTests(BaseCheck):
|
|
827
1018
|
"""Models must have more than the specified number of unit tests.
|
|
828
1019
|
|
|
829
1020
|
Parameters:
|
|
830
|
-
min_number_of_unit_tests (
|
|
1021
|
+
min_number_of_unit_tests (int | None): The minimum number of unit tests that a model must have.
|
|
831
1022
|
|
|
832
1023
|
Receives:
|
|
833
1024
|
manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.
|
|
834
1025
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
835
|
-
unit_tests (
|
|
1026
|
+
unit_tests (list[UnitTests]): List of UnitTests objects parsed from `manifest.json`.
|
|
836
1027
|
|
|
837
1028
|
Other Parameters:
|
|
838
|
-
description (
|
|
839
|
-
exclude (
|
|
840
|
-
include (
|
|
841
|
-
materialization (
|
|
842
|
-
severity (
|
|
1029
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1030
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1031
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1032
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1033
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
843
1034
|
|
|
844
1035
|
!!! warning
|
|
845
1036
|
|
|
@@ -859,27 +1050,39 @@ class CheckModelHasUnitTests(BaseCheck):
|
|
|
859
1050
|
|
|
860
1051
|
"""
|
|
861
1052
|
|
|
862
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
1053
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
863
1054
|
min_number_of_unit_tests: int = Field(default=1)
|
|
864
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1055
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
865
1056
|
name: Literal["check_model_has_unit_tests"]
|
|
866
|
-
unit_tests:
|
|
1057
|
+
unit_tests: list["UnitTests"] = Field(default=[])
|
|
867
1058
|
|
|
868
1059
|
def execute(self) -> None:
|
|
869
|
-
"""Execute the check.
|
|
1060
|
+
"""Execute the check.
|
|
1061
|
+
|
|
1062
|
+
Raises:
|
|
1063
|
+
DbtBouncerFailedCheckError: If model does not have enough unit tests.
|
|
1064
|
+
|
|
1065
|
+
"""
|
|
1066
|
+
if self.manifest_obj is None:
|
|
1067
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
1068
|
+
if self.model is None:
|
|
1069
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
870
1070
|
if get_package_version_number(
|
|
871
|
-
self.manifest_obj.manifest.metadata.dbt_version
|
|
1071
|
+
self.manifest_obj.manifest.metadata.dbt_version or "0.0.0"
|
|
872
1072
|
) >= get_package_version_number("1.8.0"):
|
|
873
1073
|
num_unit_tests = len(
|
|
874
1074
|
[
|
|
875
1075
|
t.unique_id
|
|
876
1076
|
for t in self.unit_tests
|
|
877
|
-
if t.depends_on
|
|
1077
|
+
if t.depends_on
|
|
1078
|
+
and t.depends_on.nodes
|
|
1079
|
+
and t.depends_on.nodes[0] == self.model.unique_id
|
|
878
1080
|
],
|
|
879
1081
|
)
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
1082
|
+
if num_unit_tests < self.min_number_of_unit_tests:
|
|
1083
|
+
raise DbtBouncerFailedCheckError(
|
|
1084
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {num_unit_tests} unit tests, this is less than the minimum of {self.min_number_of_unit_tests}."
|
|
1085
|
+
)
|
|
883
1086
|
else:
|
|
884
1087
|
logging.warning(
|
|
885
1088
|
"The `check_model_has_unit_tests` check is only supported for dbt 1.8.0 and above.",
|
|
@@ -893,11 +1096,11 @@ class CheckModelLatestVersionSpecified(BaseCheck):
|
|
|
893
1096
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
894
1097
|
|
|
895
1098
|
Other Parameters:
|
|
896
|
-
description (
|
|
897
|
-
exclude (
|
|
898
|
-
include (
|
|
899
|
-
materialization (
|
|
900
|
-
severity (
|
|
1099
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1100
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1101
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1102
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1103
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
901
1104
|
|
|
902
1105
|
Example(s):
|
|
903
1106
|
```yaml
|
|
@@ -910,33 +1113,41 @@ class CheckModelLatestVersionSpecified(BaseCheck):
|
|
|
910
1113
|
|
|
911
1114
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
912
1115
|
|
|
913
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1116
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
914
1117
|
name: Literal["check_model_latest_version_specified"]
|
|
915
1118
|
|
|
916
1119
|
def execute(self) -> None:
|
|
917
|
-
"""Execute the check.
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1120
|
+
"""Execute the check.
|
|
1121
|
+
|
|
1122
|
+
Raises:
|
|
1123
|
+
DbtBouncerFailedCheckError: If latest version is not specified.
|
|
1124
|
+
|
|
1125
|
+
"""
|
|
1126
|
+
if self.model is None:
|
|
1127
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1128
|
+
if self.model.latest_version is None:
|
|
1129
|
+
raise DbtBouncerFailedCheckError(
|
|
1130
|
+
f"`{self.model.name}` does not have a specified `latest_version`."
|
|
1131
|
+
)
|
|
921
1132
|
|
|
922
1133
|
|
|
923
1134
|
class CheckModelMaxChainedViews(BaseCheck):
|
|
924
1135
|
"""Models cannot have more than the specified number of upstream dependents that are not tables.
|
|
925
1136
|
|
|
926
1137
|
Parameters:
|
|
927
|
-
materializations_to_include (
|
|
928
|
-
max_chained_views (
|
|
1138
|
+
materializations_to_include (list[str] | None): List of materializations to include in the check.
|
|
1139
|
+
max_chained_views (int | None): The maximum number of upstream dependents that are not tables.
|
|
929
1140
|
|
|
930
1141
|
Receives:
|
|
931
1142
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
932
|
-
models (
|
|
1143
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
933
1144
|
|
|
934
1145
|
Other Parameters:
|
|
935
|
-
description (
|
|
936
|
-
exclude (
|
|
937
|
-
include (
|
|
938
|
-
materialization (
|
|
939
|
-
severity (
|
|
1146
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1147
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1148
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1149
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1150
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
940
1151
|
|
|
941
1152
|
Example(s):
|
|
942
1153
|
```yaml
|
|
@@ -955,20 +1166,29 @@ class CheckModelMaxChainedViews(BaseCheck):
|
|
|
955
1166
|
|
|
956
1167
|
"""
|
|
957
1168
|
|
|
958
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
959
|
-
materializations_to_include:
|
|
1169
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
1170
|
+
materializations_to_include: list[str] = Field(
|
|
960
1171
|
default=["ephemeral", "view"],
|
|
961
1172
|
)
|
|
962
1173
|
max_chained_views: int = Field(
|
|
963
1174
|
default=3,
|
|
964
1175
|
)
|
|
965
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
966
|
-
models:
|
|
1176
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1177
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
967
1178
|
name: Literal["check_model_max_chained_views"]
|
|
968
|
-
package_name:
|
|
1179
|
+
package_name: str | None = Field(default=None)
|
|
969
1180
|
|
|
970
1181
|
def execute(self) -> None:
|
|
971
|
-
"""Execute the check.
|
|
1182
|
+
"""Execute the check.
|
|
1183
|
+
|
|
1184
|
+
Raises:
|
|
1185
|
+
DbtBouncerFailedCheckError: If max chained views exceeded.
|
|
1186
|
+
|
|
1187
|
+
"""
|
|
1188
|
+
if self.model is None:
|
|
1189
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1190
|
+
if self.manifest_obj is None:
|
|
1191
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
972
1192
|
|
|
973
1193
|
def return_upstream_view_models(
|
|
974
1194
|
materializations,
|
|
@@ -982,7 +1202,7 @@ class CheckModelMaxChainedViews(BaseCheck):
|
|
|
982
1202
|
|
|
983
1203
|
Returns
|
|
984
1204
|
-
|
|
985
|
-
|
|
1205
|
+
list[str]: List of model unique_id's of upstream models that are views.
|
|
986
1206
|
|
|
987
1207
|
"""
|
|
988
1208
|
if depth == max_chained_views or model_unique_ids_to_check == []:
|
|
@@ -1019,7 +1239,7 @@ class CheckModelMaxChainedViews(BaseCheck):
|
|
|
1019
1239
|
depth=depth,
|
|
1020
1240
|
)
|
|
1021
1241
|
|
|
1022
|
-
|
|
1242
|
+
if (
|
|
1023
1243
|
len(
|
|
1024
1244
|
return_upstream_view_models(
|
|
1025
1245
|
materializations=self.materializations_to_include,
|
|
@@ -1032,28 +1252,29 @@ class CheckModelMaxChainedViews(BaseCheck):
|
|
|
1032
1252
|
),
|
|
1033
1253
|
),
|
|
1034
1254
|
)
|
|
1035
|
-
|
|
1036
|
-
)
|
|
1037
|
-
|
|
1038
|
-
|
|
1255
|
+
!= 0
|
|
1256
|
+
):
|
|
1257
|
+
raise DbtBouncerFailedCheckError(
|
|
1258
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has more than {self.max_chained_views} upstream dependents that are not tables."
|
|
1259
|
+
)
|
|
1039
1260
|
|
|
1040
1261
|
|
|
1041
1262
|
class CheckModelMaxFanout(BaseCheck):
|
|
1042
1263
|
"""Models cannot have more than the specified number of downstream models.
|
|
1043
1264
|
|
|
1044
1265
|
Parameters:
|
|
1045
|
-
max_downstream_models (
|
|
1266
|
+
max_downstream_models (int | None): The maximum number of permitted downstream models.
|
|
1046
1267
|
|
|
1047
1268
|
Receives:
|
|
1048
1269
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1049
|
-
models (
|
|
1270
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
1050
1271
|
|
|
1051
1272
|
Other Parameters:
|
|
1052
|
-
description (
|
|
1053
|
-
exclude (
|
|
1054
|
-
include (
|
|
1055
|
-
materialization (
|
|
1056
|
-
severity (
|
|
1273
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1274
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1275
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1276
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1277
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1057
1278
|
|
|
1058
1279
|
Example(s):
|
|
1059
1280
|
```yaml
|
|
@@ -1065,19 +1286,29 @@ class CheckModelMaxFanout(BaseCheck):
|
|
|
1065
1286
|
"""
|
|
1066
1287
|
|
|
1067
1288
|
max_downstream_models: int = Field(default=3)
|
|
1068
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1069
|
-
models:
|
|
1289
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1290
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
1070
1291
|
name: Literal["check_model_max_fanout"]
|
|
1071
1292
|
|
|
1072
1293
|
def execute(self) -> None:
|
|
1073
|
-
"""Execute the check.
|
|
1294
|
+
"""Execute the check.
|
|
1295
|
+
|
|
1296
|
+
Raises:
|
|
1297
|
+
DbtBouncerFailedCheckError: If max fanout exceeded.
|
|
1298
|
+
|
|
1299
|
+
"""
|
|
1300
|
+
if self.model is None:
|
|
1301
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1074
1302
|
num_downstream_models = sum(
|
|
1075
|
-
self.model.unique_id
|
|
1303
|
+
self.model.unique_id
|
|
1304
|
+
in (getattr(m.depends_on, "nodes", []) if m.depends_on else [])
|
|
1305
|
+
for m in self.models
|
|
1076
1306
|
)
|
|
1077
1307
|
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1308
|
+
if num_downstream_models > self.max_downstream_models:
|
|
1309
|
+
raise DbtBouncerFailedCheckError(
|
|
1310
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {num_downstream_models} downstream models, which is more than the permitted maximum of {self.max_downstream_models}."
|
|
1311
|
+
)
|
|
1081
1312
|
|
|
1082
1313
|
|
|
1083
1314
|
class CheckModelMaxNumberOfLines(BaseCheck):
|
|
@@ -1089,11 +1320,11 @@ class CheckModelMaxNumberOfLines(BaseCheck):
|
|
|
1089
1320
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1090
1321
|
|
|
1091
1322
|
Other Parameters:
|
|
1092
|
-
description (
|
|
1093
|
-
exclude (
|
|
1094
|
-
include (
|
|
1095
|
-
materialization (
|
|
1096
|
-
severity (
|
|
1323
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1324
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1325
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1326
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1327
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1097
1328
|
|
|
1098
1329
|
Example(s):
|
|
1099
1330
|
```yaml
|
|
@@ -1108,36 +1339,44 @@ class CheckModelMaxNumberOfLines(BaseCheck):
|
|
|
1108
1339
|
|
|
1109
1340
|
"""
|
|
1110
1341
|
|
|
1111
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1342
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1112
1343
|
name: Literal["check_model_max_number_of_lines"]
|
|
1113
1344
|
max_number_of_lines: int = Field(default=100)
|
|
1114
1345
|
|
|
1115
1346
|
def execute(self) -> None:
|
|
1116
|
-
"""Execute the check.
|
|
1117
|
-
actual_number_of_lines = self.model.raw_code.count("\n") + 1
|
|
1347
|
+
"""Execute the check.
|
|
1118
1348
|
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1349
|
+
Raises:
|
|
1350
|
+
DbtBouncerFailedCheckError: If max lines exceeded.
|
|
1351
|
+
|
|
1352
|
+
"""
|
|
1353
|
+
if self.model is None:
|
|
1354
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1355
|
+
actual_number_of_lines = (self.model.raw_code or "").count("\n") + 1
|
|
1356
|
+
|
|
1357
|
+
if actual_number_of_lines > self.max_number_of_lines:
|
|
1358
|
+
raise DbtBouncerFailedCheckError(
|
|
1359
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {actual_number_of_lines} lines, this is more than the maximum permitted number of lines ({self.max_number_of_lines})."
|
|
1360
|
+
)
|
|
1122
1361
|
|
|
1123
1362
|
|
|
1124
1363
|
class CheckModelMaxUpstreamDependencies(BaseCheck):
|
|
1125
1364
|
"""Limit the number of upstream dependencies a model has.
|
|
1126
1365
|
|
|
1127
1366
|
Parameters:
|
|
1128
|
-
max_upstream_macros (
|
|
1129
|
-
max_upstream_models (
|
|
1130
|
-
max_upstream_sources (
|
|
1367
|
+
max_upstream_macros (int | None): The maximum number of permitted upstream macros.
|
|
1368
|
+
max_upstream_models (int | None): The maximum number of permitted upstream models.
|
|
1369
|
+
max_upstream_sources (int | None): The maximum number of permitted upstream sources.
|
|
1131
1370
|
|
|
1132
1371
|
Receives:
|
|
1133
1372
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1134
1373
|
|
|
1135
1374
|
Other Parameters:
|
|
1136
|
-
description (
|
|
1137
|
-
exclude (
|
|
1138
|
-
include (
|
|
1139
|
-
materialization (
|
|
1140
|
-
severity (
|
|
1375
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1376
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1377
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1378
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1379
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1141
1380
|
|
|
1142
1381
|
Example(s):
|
|
1143
1382
|
```yaml
|
|
@@ -1157,28 +1396,45 @@ class CheckModelMaxUpstreamDependencies(BaseCheck):
|
|
|
1157
1396
|
max_upstream_sources: int = Field(
|
|
1158
1397
|
default=1,
|
|
1159
1398
|
)
|
|
1160
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1399
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1161
1400
|
name: Literal["check_model_max_upstream_dependencies"]
|
|
1162
1401
|
|
|
1163
1402
|
def execute(self) -> None:
|
|
1164
|
-
"""Execute the check.
|
|
1165
|
-
num_upstream_macros = len(list(self.model.depends_on.macros))
|
|
1166
|
-
num_upstream_models = len(
|
|
1167
|
-
[m for m in self.model.depends_on.nodes if m.split(".")[0] == "model"],
|
|
1168
|
-
)
|
|
1169
|
-
num_upstream_sources = len(
|
|
1170
|
-
[m for m in self.model.depends_on.nodes if m.split(".")[0] == "source"],
|
|
1171
|
-
)
|
|
1403
|
+
"""Execute the check.
|
|
1172
1404
|
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1405
|
+
Raises:
|
|
1406
|
+
DbtBouncerFailedCheckError: If max upstream dependencies exceeded.
|
|
1407
|
+
|
|
1408
|
+
"""
|
|
1409
|
+
if self.model is None:
|
|
1410
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1411
|
+
depends_on = self.model.depends_on
|
|
1412
|
+
if depends_on:
|
|
1413
|
+
num_upstream_macros = len(list(getattr(depends_on, "macros", []) or []))
|
|
1414
|
+
nodes = getattr(depends_on, "nodes", []) or []
|
|
1415
|
+
num_upstream_models = len(
|
|
1416
|
+
[m for m in nodes if m.split(".")[0] == "model"],
|
|
1417
|
+
)
|
|
1418
|
+
num_upstream_sources = len(
|
|
1419
|
+
[m for m in nodes if m.split(".")[0] == "source"],
|
|
1420
|
+
)
|
|
1421
|
+
else:
|
|
1422
|
+
num_upstream_macros = 0
|
|
1423
|
+
num_upstream_models = 0
|
|
1424
|
+
num_upstream_sources = 0
|
|
1425
|
+
|
|
1426
|
+
if num_upstream_macros > self.max_upstream_macros:
|
|
1427
|
+
raise DbtBouncerFailedCheckError(
|
|
1428
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {num_upstream_macros} upstream macros, which is more than the permitted maximum of {self.max_upstream_macros}."
|
|
1429
|
+
)
|
|
1430
|
+
if num_upstream_models > self.max_upstream_models:
|
|
1431
|
+
raise DbtBouncerFailedCheckError(
|
|
1432
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {num_upstream_models} upstream models, which is more than the permitted maximum of {self.max_upstream_models}."
|
|
1433
|
+
)
|
|
1434
|
+
if num_upstream_sources > self.max_upstream_sources:
|
|
1435
|
+
raise DbtBouncerFailedCheckError(
|
|
1436
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has {num_upstream_sources} upstream sources, which is more than the permitted maximum of {self.max_upstream_sources}."
|
|
1437
|
+
)
|
|
1182
1438
|
|
|
1183
1439
|
|
|
1184
1440
|
class CheckModelNames(BaseCheck):
|
|
@@ -1191,11 +1447,11 @@ class CheckModelNames(BaseCheck):
|
|
|
1191
1447
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1192
1448
|
|
|
1193
1449
|
Other Parameters:
|
|
1194
|
-
description (
|
|
1195
|
-
exclude (
|
|
1196
|
-
include (
|
|
1197
|
-
materialization (
|
|
1198
|
-
severity (
|
|
1450
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1451
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1452
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1453
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1454
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1199
1455
|
|
|
1200
1456
|
Example(s):
|
|
1201
1457
|
```yaml
|
|
@@ -1212,34 +1468,42 @@ class CheckModelNames(BaseCheck):
|
|
|
1212
1468
|
|
|
1213
1469
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
1214
1470
|
|
|
1215
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1471
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1216
1472
|
name: Literal["check_model_names"]
|
|
1217
1473
|
model_name_pattern: str
|
|
1218
1474
|
|
|
1219
1475
|
def execute(self) -> None:
|
|
1220
|
-
"""Execute the check.
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1476
|
+
"""Execute the check.
|
|
1477
|
+
|
|
1478
|
+
Raises:
|
|
1479
|
+
DbtBouncerFailedCheckError: If model name does not match regex.
|
|
1480
|
+
|
|
1481
|
+
"""
|
|
1482
|
+
if self.model is None:
|
|
1483
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1484
|
+
if (
|
|
1485
|
+
re.compile(self.model_name_pattern.strip()).match(str(self.model.name))
|
|
1486
|
+
is None
|
|
1487
|
+
):
|
|
1488
|
+
raise DbtBouncerFailedCheckError(
|
|
1489
|
+
f"`{get_clean_model_name(self.model.unique_id)}` does not match the supplied regex `{self.model_name_pattern.strip()}`."
|
|
1490
|
+
)
|
|
1227
1491
|
|
|
1228
1492
|
|
|
1229
1493
|
class CheckModelNumberOfGrants(BaseCheck):
|
|
1230
1494
|
"""Model can have the specified number of privileges.
|
|
1231
1495
|
|
|
1232
1496
|
Receives:
|
|
1233
|
-
max_number_of_privileges (
|
|
1234
|
-
min_number_of_privileges (
|
|
1497
|
+
max_number_of_privileges (int | None): Maximum number of privileges, inclusive.
|
|
1498
|
+
min_number_of_privileges (int | None): Minimum number of privileges, inclusive.
|
|
1235
1499
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1236
1500
|
|
|
1237
1501
|
Other Parameters:
|
|
1238
|
-
description (
|
|
1239
|
-
exclude (
|
|
1240
|
-
include (
|
|
1241
|
-
materialization (
|
|
1242
|
-
severity (
|
|
1502
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1503
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1504
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1505
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1506
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1243
1507
|
|
|
1244
1508
|
Example(s):
|
|
1245
1509
|
```yaml
|
|
@@ -1252,21 +1516,32 @@ class CheckModelNumberOfGrants(BaseCheck):
|
|
|
1252
1516
|
|
|
1253
1517
|
"""
|
|
1254
1518
|
|
|
1255
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1519
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1256
1520
|
name: Literal["check_model_number_of_grants"]
|
|
1257
1521
|
max_number_of_privileges: int = Field(default=100)
|
|
1258
1522
|
min_number_of_privileges: int = Field(default=0)
|
|
1259
1523
|
|
|
1260
1524
|
def execute(self) -> None:
|
|
1261
|
-
"""Execute the check.
|
|
1262
|
-
num_grants = len(self.model.config.grants.keys())
|
|
1525
|
+
"""Execute the check.
|
|
1263
1526
|
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1527
|
+
Raises:
|
|
1528
|
+
DbtBouncerFailedCheckError: If number of grants is not within limits.
|
|
1529
|
+
|
|
1530
|
+
"""
|
|
1531
|
+
if self.model is None:
|
|
1532
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1533
|
+
config = self.model.config
|
|
1534
|
+
grants = config.grants if config else {}
|
|
1535
|
+
num_grants = len((grants or {}).keys())
|
|
1536
|
+
|
|
1537
|
+
if num_grants < self.min_number_of_privileges:
|
|
1538
|
+
raise DbtBouncerFailedCheckError(
|
|
1539
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has less grants (`{num_grants}`) than the specified minimum ({self.min_number_of_privileges})."
|
|
1540
|
+
)
|
|
1541
|
+
if num_grants > self.max_number_of_privileges:
|
|
1542
|
+
raise DbtBouncerFailedCheckError(
|
|
1543
|
+
f"`{get_clean_model_name(self.model.unique_id)}` has more grants (`{num_grants}`) than the specified maximum ({self.max_number_of_privileges})."
|
|
1544
|
+
)
|
|
1270
1545
|
|
|
1271
1546
|
|
|
1272
1547
|
class CheckModelPropertyFileLocation(BaseCheck):
|
|
@@ -1276,11 +1551,11 @@ class CheckModelPropertyFileLocation(BaseCheck):
|
|
|
1276
1551
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1277
1552
|
|
|
1278
1553
|
Other Parameters:
|
|
1279
|
-
description (
|
|
1280
|
-
exclude (
|
|
1281
|
-
include (
|
|
1282
|
-
materialization (
|
|
1283
|
-
severity (
|
|
1554
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1555
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1556
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1557
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1558
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1284
1559
|
|
|
1285
1560
|
Example(s):
|
|
1286
1561
|
```yaml
|
|
@@ -1290,37 +1565,56 @@ class CheckModelPropertyFileLocation(BaseCheck):
|
|
|
1290
1565
|
|
|
1291
1566
|
"""
|
|
1292
1567
|
|
|
1293
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1568
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1294
1569
|
name: Literal["check_model_property_file_location"]
|
|
1295
1570
|
|
|
1296
1571
|
def execute(self) -> None:
|
|
1297
|
-
"""Execute the check.
|
|
1298
|
-
|
|
1572
|
+
"""Execute the check.
|
|
1573
|
+
|
|
1574
|
+
Raises:
|
|
1575
|
+
DbtBouncerFailedCheckError: If property file location is incorrect.
|
|
1576
|
+
|
|
1577
|
+
"""
|
|
1578
|
+
if self.model is None:
|
|
1579
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1580
|
+
if not (
|
|
1299
1581
|
hasattr(self.model, "patch_path")
|
|
1300
|
-
and
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
.replace("intermediate", "int")
|
|
1307
|
-
.replace("marts", "")
|
|
1308
|
-
)
|
|
1309
|
-
properties_yml_name = clean_path_str(self.model.patch_path).split("/")[-1]
|
|
1582
|
+
and self.model.patch_path
|
|
1583
|
+
and clean_path_str(self.model.patch_path or "") is not None
|
|
1584
|
+
):
|
|
1585
|
+
raise DbtBouncerFailedCheckError(
|
|
1586
|
+
f"`{get_clean_model_name(self.model.unique_id)}` is not documented."
|
|
1587
|
+
)
|
|
1310
1588
|
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
"
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1589
|
+
original_path = Path(clean_path_str(self.model.original_file_path))
|
|
1590
|
+
relevant_parts = original_path.parts[1:-1]
|
|
1591
|
+
|
|
1592
|
+
mapped_parts = []
|
|
1593
|
+
for part in relevant_parts:
|
|
1594
|
+
if part == "staging":
|
|
1595
|
+
mapped_parts.append("stg")
|
|
1596
|
+
elif part == "intermediate":
|
|
1597
|
+
mapped_parts.append("int")
|
|
1598
|
+
elif part == "marts":
|
|
1599
|
+
continue
|
|
1600
|
+
else:
|
|
1601
|
+
mapped_parts.append(part)
|
|
1602
|
+
|
|
1603
|
+
expected_substr = "_".join(mapped_parts)
|
|
1604
|
+
properties_yml_name = Path(clean_path_str(self.model.patch_path or "")).name
|
|
1605
|
+
|
|
1606
|
+
if not properties_yml_name.startswith("_"):
|
|
1607
|
+
raise DbtBouncerFailedCheckError(
|
|
1608
|
+
f"The properties file for `{get_clean_model_name(self.model.unique_id)}` (`{properties_yml_name}`) does not start with an underscore."
|
|
1609
|
+
)
|
|
1610
|
+
if expected_substr not in properties_yml_name:
|
|
1611
|
+
raise DbtBouncerFailedCheckError(
|
|
1612
|
+
f"The properties file for `{get_clean_model_name(self.model.unique_id)}` (`{properties_yml_name}`) does not contain the expected substring (`{expected_substr}`)."
|
|
1613
|
+
)
|
|
1614
|
+
if not properties_yml_name.endswith("__models.yml"):
|
|
1615
|
+
raise DbtBouncerFailedCheckError(
|
|
1616
|
+
f"The properties file for `{get_clean_model_name(self.model.unique_id)}` (`{properties_yml_name}`) does not end with `__models.yml`."
|
|
1617
|
+
)
|
|
1324
1618
|
|
|
1325
1619
|
|
|
1326
1620
|
class CheckModelSchemaName(BaseCheck):
|
|
@@ -1339,11 +1633,11 @@ class CheckModelSchemaName(BaseCheck):
|
|
|
1339
1633
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1340
1634
|
|
|
1341
1635
|
Other Parameters:
|
|
1342
|
-
description (
|
|
1343
|
-
exclude (
|
|
1344
|
-
include (
|
|
1345
|
-
materialization (
|
|
1346
|
-
severity (
|
|
1636
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1637
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1638
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1639
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1640
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1347
1641
|
|
|
1348
1642
|
Example(s):
|
|
1349
1643
|
```yaml
|
|
@@ -1360,18 +1654,26 @@ class CheckModelSchemaName(BaseCheck):
|
|
|
1360
1654
|
|
|
1361
1655
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
1362
1656
|
|
|
1363
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1657
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1364
1658
|
name: Literal["check_model_schema_name"]
|
|
1365
1659
|
schema_name_pattern: str
|
|
1366
1660
|
|
|
1367
1661
|
def execute(self) -> None:
|
|
1368
|
-
"""Execute the check.
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1662
|
+
"""Execute the check.
|
|
1663
|
+
|
|
1664
|
+
Raises:
|
|
1665
|
+
DbtBouncerFailedCheckError: If schema name does not match regex.
|
|
1666
|
+
|
|
1667
|
+
"""
|
|
1668
|
+
if self.model is None:
|
|
1669
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1670
|
+
if (
|
|
1671
|
+
re.compile(self.schema_name_pattern.strip()).match(str(self.model.schema_))
|
|
1672
|
+
is None
|
|
1673
|
+
):
|
|
1674
|
+
raise DbtBouncerFailedCheckError(
|
|
1675
|
+
f"`{self.model.schema_}` does not match the supplied regex `{self.schema_name_pattern.strip()})`."
|
|
1676
|
+
)
|
|
1375
1677
|
|
|
1376
1678
|
|
|
1377
1679
|
class CheckModelVersionAllowed(BaseCheck):
|
|
@@ -1384,11 +1686,11 @@ class CheckModelVersionAllowed(BaseCheck):
|
|
|
1384
1686
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1385
1687
|
|
|
1386
1688
|
Other Parameters:
|
|
1387
|
-
description (
|
|
1388
|
-
exclude (
|
|
1389
|
-
include (
|
|
1390
|
-
materialization (
|
|
1391
|
-
severity (
|
|
1689
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1690
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1691
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1692
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1693
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1392
1694
|
|
|
1393
1695
|
Example(s):
|
|
1394
1696
|
```yaml
|
|
@@ -1406,17 +1708,24 @@ class CheckModelVersionAllowed(BaseCheck):
|
|
|
1406
1708
|
|
|
1407
1709
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
1408
1710
|
|
|
1409
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1711
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1410
1712
|
name: Literal["check_model_version_allowed"]
|
|
1411
1713
|
version_pattern: str
|
|
1412
1714
|
|
|
1413
1715
|
def execute(self) -> None:
|
|
1414
|
-
"""Execute the check.
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1716
|
+
"""Execute the check.
|
|
1717
|
+
|
|
1718
|
+
Raises:
|
|
1719
|
+
DbtBouncerFailedCheckError: If version is not allowed.
|
|
1720
|
+
|
|
1721
|
+
"""
|
|
1722
|
+
if self.model is None:
|
|
1723
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1724
|
+
if self.model.version and (
|
|
1725
|
+
re.compile(self.version_pattern.strip()).match(str(self.model.version))
|
|
1726
|
+
is None
|
|
1727
|
+
):
|
|
1728
|
+
raise DbtBouncerFailedCheckError(
|
|
1420
1729
|
f"Version `{self.model.version}` in `{self.model.name}` does not match the supplied regex `{self.version_pattern.strip()})`."
|
|
1421
1730
|
)
|
|
1422
1731
|
|
|
@@ -1429,11 +1738,11 @@ class CheckModelVersionPinnedInRef(BaseCheck):
|
|
|
1429
1738
|
model (DbtBouncerModelBase): The DbtBouncerModelBase object to check.
|
|
1430
1739
|
|
|
1431
1740
|
Other Parameters:
|
|
1432
|
-
description (
|
|
1433
|
-
exclude (
|
|
1434
|
-
include (
|
|
1435
|
-
materialization (
|
|
1436
|
-
severity (
|
|
1741
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1742
|
+
exclude (str | None): Regex pattern to match the model path. Model paths that match the pattern will not be checked.
|
|
1743
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
1744
|
+
materialization (Literal["ephemeral", "incremental", "table", "view"] | None): Limit check to models with the specified materialization.
|
|
1745
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1437
1746
|
|
|
1438
1747
|
Example(s):
|
|
1439
1748
|
```yaml
|
|
@@ -1446,42 +1755,60 @@ class CheckModelVersionPinnedInRef(BaseCheck):
|
|
|
1446
1755
|
|
|
1447
1756
|
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
1448
1757
|
|
|
1449
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
1450
|
-
model: "DbtBouncerModelBase" = Field(default=None)
|
|
1758
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
1759
|
+
model: "DbtBouncerModelBase | None" = Field(default=None)
|
|
1451
1760
|
name: Literal["check_model_version_pinned_in_ref"]
|
|
1452
1761
|
|
|
1453
1762
|
def execute(self) -> None:
|
|
1454
|
-
"""Execute the check.
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1763
|
+
"""Execute the check.
|
|
1764
|
+
|
|
1765
|
+
Raises:
|
|
1766
|
+
DbtBouncerFailedCheckError: If version is not pinned in ref.
|
|
1767
|
+
|
|
1768
|
+
"""
|
|
1769
|
+
if self.model is None:
|
|
1770
|
+
raise DbtBouncerFailedCheckError("self.model is None")
|
|
1771
|
+
if self.manifest_obj is None:
|
|
1772
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
1773
|
+
child_map = self.manifest_obj.manifest.child_map
|
|
1774
|
+
if child_map and self.model.unique_id in child_map:
|
|
1775
|
+
downstream_models = [
|
|
1776
|
+
x for x in child_map[self.model.unique_id] if x.startswith("model.")
|
|
1777
|
+
]
|
|
1778
|
+
else:
|
|
1779
|
+
downstream_models = []
|
|
1780
|
+
|
|
1781
|
+
downstream_models_with_unversioned_refs: list[str] = []
|
|
1461
1782
|
for m in downstream_models:
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1783
|
+
node = self.manifest_obj.manifest.nodes.get(m)
|
|
1784
|
+
refs = getattr(node, "refs", None)
|
|
1785
|
+
if node and refs and isinstance(refs, list):
|
|
1786
|
+
downstream_models_with_unversioned_refs.extend(
|
|
1787
|
+
m
|
|
1788
|
+
for ref in refs
|
|
1789
|
+
if getattr(ref, "name", None) == self.model.unique_id.split(".")[-1]
|
|
1790
|
+
and not getattr(ref, "version", None)
|
|
1791
|
+
)
|
|
1467
1792
|
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1793
|
+
if downstream_models_with_unversioned_refs:
|
|
1794
|
+
raise DbtBouncerFailedCheckError(
|
|
1795
|
+
f"`{self.model.name}` is referenced without a pinned version in downstream models: {downstream_models_with_unversioned_refs}."
|
|
1796
|
+
)
|
|
1471
1797
|
|
|
1472
1798
|
|
|
1473
1799
|
class CheckModelsDocumentationCoverage(BaseModel):
|
|
1474
1800
|
"""Set the minimum percentage of models that have a populated description.
|
|
1475
1801
|
|
|
1476
1802
|
Parameters:
|
|
1803
|
+
min_description_length (int | None): Minimum length required for the description to be considered populated.
|
|
1477
1804
|
min_model_documentation_coverage_pct (float): The minimum percentage of models that must have a populated description.
|
|
1478
1805
|
|
|
1479
1806
|
Receives:
|
|
1480
|
-
models (
|
|
1807
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
1481
1808
|
|
|
1482
1809
|
Other Parameters:
|
|
1483
|
-
description (
|
|
1484
|
-
severity (
|
|
1810
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1811
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1485
1812
|
|
|
1486
1813
|
Example(s):
|
|
1487
1814
|
```yaml
|
|
@@ -1489,16 +1816,21 @@ class CheckModelsDocumentationCoverage(BaseModel):
|
|
|
1489
1816
|
- name: check_model_documentation_coverage
|
|
1490
1817
|
min_model_documentation_coverage_pct: 90
|
|
1491
1818
|
```
|
|
1819
|
+
```yaml
|
|
1820
|
+
manifest_checks:
|
|
1821
|
+
- name: check_model_documentation_coverage
|
|
1822
|
+
min_description_length: 25 # Setting a stricter requirement for description length
|
|
1823
|
+
```
|
|
1492
1824
|
|
|
1493
1825
|
"""
|
|
1494
1826
|
|
|
1495
1827
|
model_config = ConfigDict(extra="forbid")
|
|
1496
1828
|
|
|
1497
|
-
description:
|
|
1829
|
+
description: str | None = Field(
|
|
1498
1830
|
default=None,
|
|
1499
1831
|
description="Description of what the check does and why it is implemented.",
|
|
1500
1832
|
)
|
|
1501
|
-
index:
|
|
1833
|
+
index: int | None = Field(
|
|
1502
1834
|
default=None,
|
|
1503
1835
|
description="Index to uniquely identify the check, calculated at runtime.",
|
|
1504
1836
|
)
|
|
@@ -1507,19 +1839,26 @@ class CheckModelsDocumentationCoverage(BaseModel):
|
|
|
1507
1839
|
ge=0,
|
|
1508
1840
|
le=100,
|
|
1509
1841
|
)
|
|
1510
|
-
models:
|
|
1842
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
1511
1843
|
name: Literal["check_model_documentation_coverage"]
|
|
1512
|
-
severity:
|
|
1844
|
+
severity: Literal["error", "warn"] | None = Field(
|
|
1513
1845
|
default="error",
|
|
1514
1846
|
description="Severity of the check, one of 'error' or 'warn'.",
|
|
1515
1847
|
)
|
|
1516
1848
|
|
|
1517
1849
|
def execute(self) -> None:
|
|
1518
|
-
"""Execute the check.
|
|
1850
|
+
"""Execute the check.
|
|
1851
|
+
|
|
1852
|
+
Raises:
|
|
1853
|
+
DbtBouncerFailedCheckError: If documentation coverage is less than minimum.
|
|
1854
|
+
|
|
1855
|
+
"""
|
|
1519
1856
|
num_models = len(self.models)
|
|
1520
1857
|
models_with_description = []
|
|
1521
1858
|
for model in self.models:
|
|
1522
|
-
if
|
|
1859
|
+
if is_description_populated(
|
|
1860
|
+
description=model.description or "", min_description_length=4
|
|
1861
|
+
):
|
|
1523
1862
|
models_with_description.append(model.unique_id)
|
|
1524
1863
|
|
|
1525
1864
|
num_models_with_descriptions = len(models_with_description)
|
|
@@ -1527,11 +1866,10 @@ class CheckModelsDocumentationCoverage(BaseModel):
|
|
|
1527
1866
|
num_models_with_descriptions / num_models
|
|
1528
1867
|
) * 100
|
|
1529
1868
|
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
)
|
|
1869
|
+
if model_description_coverage_pct < self.min_model_documentation_coverage_pct:
|
|
1870
|
+
raise DbtBouncerFailedCheckError(
|
|
1871
|
+
f"Only {model_description_coverage_pct}% of models have a populated description, this is less than the permitted minimum of {self.min_model_documentation_coverage_pct}%."
|
|
1872
|
+
)
|
|
1535
1873
|
|
|
1536
1874
|
|
|
1537
1875
|
class CheckModelsTestCoverage(BaseModel):
|
|
@@ -1539,12 +1877,12 @@ class CheckModelsTestCoverage(BaseModel):
|
|
|
1539
1877
|
|
|
1540
1878
|
Parameters:
|
|
1541
1879
|
min_model_test_coverage_pct (float): The minimum percentage of models that must have at least one test.
|
|
1542
|
-
models (
|
|
1543
|
-
tests (
|
|
1880
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
1881
|
+
tests (list[DbtBouncerTestBase]): List of DbtBouncerTestBase objects parsed from `manifest.json`.
|
|
1544
1882
|
|
|
1545
1883
|
Other Parameters:
|
|
1546
|
-
description (
|
|
1547
|
-
severity (
|
|
1884
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
1885
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
1548
1886
|
|
|
1549
1887
|
|
|
1550
1888
|
Example(s):
|
|
@@ -1558,11 +1896,11 @@ class CheckModelsTestCoverage(BaseModel):
|
|
|
1558
1896
|
|
|
1559
1897
|
model_config = ConfigDict(extra="forbid")
|
|
1560
1898
|
|
|
1561
|
-
description:
|
|
1899
|
+
description: str | None = Field(
|
|
1562
1900
|
default=None,
|
|
1563
1901
|
description="Description of what the check does and why it is implemented.",
|
|
1564
1902
|
)
|
|
1565
|
-
index:
|
|
1903
|
+
index: int | None = Field(
|
|
1566
1904
|
default=None,
|
|
1567
1905
|
description="Index to uniquely identify the check, calculated at runtime.",
|
|
1568
1906
|
)
|
|
@@ -1572,24 +1910,32 @@ class CheckModelsTestCoverage(BaseModel):
|
|
|
1572
1910
|
ge=0,
|
|
1573
1911
|
le=100,
|
|
1574
1912
|
)
|
|
1575
|
-
models:
|
|
1576
|
-
severity:
|
|
1913
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
1914
|
+
severity: Literal["error", "warn"] | None = Field(
|
|
1577
1915
|
default="error",
|
|
1578
1916
|
description="Severity of the check, one of 'error' or 'warn'.",
|
|
1579
1917
|
)
|
|
1580
|
-
tests:
|
|
1918
|
+
tests: list["DbtBouncerTestBase"] = Field(default=[])
|
|
1581
1919
|
|
|
1582
1920
|
def execute(self) -> None:
|
|
1583
|
-
"""Execute the check.
|
|
1921
|
+
"""Execute the check.
|
|
1922
|
+
|
|
1923
|
+
Raises:
|
|
1924
|
+
DbtBouncerFailedCheckError: If test coverage is less than minimum.
|
|
1925
|
+
|
|
1926
|
+
"""
|
|
1584
1927
|
num_models = len(self.models)
|
|
1585
1928
|
models_with_tests = []
|
|
1586
1929
|
for model in self.models:
|
|
1587
1930
|
for test in self.tests:
|
|
1588
|
-
if model.unique_id in
|
|
1931
|
+
if test.depends_on and model.unique_id in getattr(
|
|
1932
|
+
test.depends_on, "nodes", []
|
|
1933
|
+
):
|
|
1589
1934
|
models_with_tests.append(model.unique_id)
|
|
1590
1935
|
num_models_with_tests = len(set(models_with_tests))
|
|
1591
1936
|
model_test_coverage_pct = (num_models_with_tests / num_models) * 100
|
|
1592
1937
|
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1938
|
+
if model_test_coverage_pct < self.min_model_test_coverage_pct:
|
|
1939
|
+
raise DbtBouncerFailedCheckError(
|
|
1940
|
+
f"Only {model_test_coverage_pct}% of models have at least one test, this is less than the permitted minimum of {self.min_model_test_coverage_pct}%."
|
|
1941
|
+
)
|