dbt-bouncer 1.31.2rc3__py3-none-any.whl → 2.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_bouncer/artifact_parsers/dbt_cloud/catalog_latest.py +21 -21
- dbt_bouncer/artifact_parsers/dbt_cloud/manifest_latest.py +1745 -1745
- dbt_bouncer/artifact_parsers/dbt_cloud/run_results_latest.py +22 -22
- dbt_bouncer/artifact_parsers/parsers_catalog.py +26 -24
- dbt_bouncer/artifact_parsers/parsers_common.py +57 -36
- dbt_bouncer/artifact_parsers/parsers_manifest.py +98 -69
- dbt_bouncer/artifact_parsers/parsers_run_results.py +32 -19
- dbt_bouncer/check_base.py +22 -11
- dbt_bouncer/checks/catalog/check_catalog_sources.py +22 -12
- dbt_bouncer/checks/catalog/check_columns.py +175 -105
- dbt_bouncer/checks/common.py +24 -3
- dbt_bouncer/checks/manifest/check_exposures.py +79 -52
- dbt_bouncer/checks/manifest/check_lineage.py +69 -40
- dbt_bouncer/checks/manifest/check_macros.py +177 -104
- dbt_bouncer/checks/manifest/check_metadata.py +28 -18
- dbt_bouncer/checks/manifest/check_models.py +842 -496
- dbt_bouncer/checks/manifest/check_seeds.py +63 -0
- dbt_bouncer/checks/manifest/check_semantic_models.py +28 -20
- dbt_bouncer/checks/manifest/check_snapshots.py +57 -33
- dbt_bouncer/checks/manifest/check_sources.py +246 -137
- dbt_bouncer/checks/manifest/check_unit_tests.py +97 -54
- dbt_bouncer/checks/run_results/check_run_results.py +34 -20
- dbt_bouncer/config_file_parser.py +47 -28
- dbt_bouncer/config_file_validator.py +11 -8
- dbt_bouncer/global_context.py +31 -0
- dbt_bouncer/main.py +128 -67
- dbt_bouncer/runner.py +61 -31
- dbt_bouncer/utils.py +146 -50
- dbt_bouncer/version.py +1 -1
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0rc1.dist-info}/METADATA +15 -15
- dbt_bouncer-2.0.0rc1.dist-info/RECORD +37 -0
- dbt_bouncer-1.31.2rc3.dist-info/RECORD +0 -35
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0rc1.dist-info}/WHEEL +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0rc1.dist-info}/entry_points.txt +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0rc1.dist-info}/licenses/LICENSE +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
# mypy: disable-error-code="union-attr"
|
|
2
1
|
import logging
|
|
3
|
-
from typing import TYPE_CHECKING,
|
|
2
|
+
from typing import TYPE_CHECKING, Literal
|
|
4
3
|
|
|
4
|
+
import pytest
|
|
5
5
|
from pydantic import BaseModel, ConfigDict, Field
|
|
6
6
|
|
|
7
7
|
from dbt_bouncer.check_base import BaseCheck
|
|
@@ -11,16 +11,14 @@ if TYPE_CHECKING:
|
|
|
11
11
|
from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
|
|
12
12
|
UnitTests,
|
|
13
13
|
)
|
|
14
|
-
from dbt_bouncer.artifact_parsers.
|
|
15
|
-
|
|
16
|
-
from dbt_bouncer.utils import object_in_path
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from dbt_bouncer.artifact_parsers.parsers_common import (
|
|
14
|
+
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
20
15
|
DbtBouncerManifest,
|
|
21
16
|
DbtBouncerModelBase,
|
|
22
17
|
)
|
|
23
18
|
|
|
19
|
+
from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
|
|
20
|
+
from dbt_bouncer.utils import object_in_path
|
|
21
|
+
|
|
24
22
|
|
|
25
23
|
class CheckUnitTestCoverage(BaseModel):
|
|
26
24
|
"""Set the minimum percentage of models that have a unit test.
|
|
@@ -33,13 +31,13 @@ class CheckUnitTestCoverage(BaseModel):
|
|
|
33
31
|
min_unit_test_coverage_pct (float): The minimum percentage of models that must have a unit test.
|
|
34
32
|
|
|
35
33
|
Receives:
|
|
36
|
-
models (
|
|
37
|
-
unit_tests (
|
|
34
|
+
models (list[DbtBouncerModelBase]): List of DbtBouncerModelBase objects parsed from `manifest.json`.
|
|
35
|
+
unit_tests (list[UnitTests]): List of UnitTests objects parsed from `manifest.json`.
|
|
38
36
|
|
|
39
37
|
Other Parameters:
|
|
40
|
-
description (
|
|
41
|
-
include (
|
|
42
|
-
severity (
|
|
38
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
39
|
+
include (str | None): Regex pattern to match the model path. Only model paths that match the pattern will be checked.
|
|
40
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
43
41
|
|
|
44
42
|
Example(s):
|
|
45
43
|
```yaml
|
|
@@ -51,36 +49,43 @@ class CheckUnitTestCoverage(BaseModel):
|
|
|
51
49
|
"""
|
|
52
50
|
|
|
53
51
|
model_config = ConfigDict(extra="forbid")
|
|
54
|
-
description:
|
|
52
|
+
description: str | None = Field(
|
|
55
53
|
default=None,
|
|
56
54
|
description="Description of what the check does and why it is implemented.",
|
|
57
55
|
)
|
|
58
|
-
include:
|
|
56
|
+
include: str | None = Field(
|
|
59
57
|
default=None,
|
|
60
58
|
description="Regexp to match which paths to include.",
|
|
61
59
|
)
|
|
62
|
-
index:
|
|
60
|
+
index: int | None = Field(
|
|
63
61
|
default=None,
|
|
64
62
|
description="Index to uniquely identify the check, calculated at runtime.",
|
|
65
63
|
)
|
|
66
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
64
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
67
65
|
min_unit_test_coverage_pct: int = Field(
|
|
68
66
|
default=100,
|
|
69
67
|
ge=0,
|
|
70
68
|
le=100,
|
|
71
69
|
)
|
|
72
|
-
models:
|
|
70
|
+
models: list["DbtBouncerModelBase"] = Field(default=[])
|
|
73
71
|
name: Literal["check_unit_test_coverage"]
|
|
74
|
-
severity:
|
|
72
|
+
severity: Literal["error", "warn"] | None = Field(
|
|
75
73
|
default="error",
|
|
76
74
|
description="Severity of the check, one of 'error' or 'warn'.",
|
|
77
75
|
)
|
|
78
|
-
unit_tests:
|
|
76
|
+
unit_tests: list["UnitTests"] = Field(default=[])
|
|
79
77
|
|
|
80
78
|
def execute(self) -> None:
|
|
81
|
-
"""Execute the check.
|
|
79
|
+
"""Execute the check.
|
|
80
|
+
|
|
81
|
+
Raises:
|
|
82
|
+
DbtBouncerFailedCheckError: If unit test coverage is less than permitted minimum.
|
|
83
|
+
|
|
84
|
+
"""
|
|
85
|
+
if self.manifest_obj is None:
|
|
86
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
82
87
|
if get_package_version_number(
|
|
83
|
-
self.manifest_obj.manifest.metadata.dbt_version
|
|
88
|
+
self.manifest_obj.manifest.metadata.dbt_version or "0.0.0"
|
|
84
89
|
) >= get_package_version_number("1.8.0"):
|
|
85
90
|
relevant_models = [
|
|
86
91
|
m.unique_id
|
|
@@ -89,18 +94,20 @@ class CheckUnitTestCoverage(BaseModel):
|
|
|
89
94
|
]
|
|
90
95
|
models_with_unit_test = []
|
|
91
96
|
for unit_test in self.unit_tests:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
97
|
+
if unit_test.depends_on and unit_test.depends_on.nodes:
|
|
98
|
+
for node in unit_test.depends_on.nodes:
|
|
99
|
+
if node in relevant_models:
|
|
100
|
+
models_with_unit_test.append(node)
|
|
95
101
|
|
|
96
102
|
num_models_with_unit_tests = len(set(models_with_unit_test))
|
|
97
103
|
unit_test_coverage_pct = (
|
|
98
104
|
num_models_with_unit_tests / len(relevant_models)
|
|
99
105
|
) * 100
|
|
100
106
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
107
|
+
if unit_test_coverage_pct < self.min_unit_test_coverage_pct:
|
|
108
|
+
raise DbtBouncerFailedCheckError(
|
|
109
|
+
f"Only {unit_test_coverage_pct}% of models have a unit test, this is less than the permitted minimum of {self.min_unit_test_coverage_pct}%."
|
|
110
|
+
)
|
|
104
111
|
else:
|
|
105
112
|
logging.warning(
|
|
106
113
|
"The `check_unit_test_expect_format` check is only supported for dbt 1.8.0 and above.",
|
|
@@ -115,17 +122,17 @@ class CheckUnitTestExpectFormats(BaseCheck):
|
|
|
115
122
|
This check is only supported for dbt 1.8.0 and above.
|
|
116
123
|
|
|
117
124
|
Parameters:
|
|
118
|
-
permitted_formats (
|
|
125
|
+
permitted_formats (list[Literal["csv", "dict", "sql"]] | None): A list of formats that are allowed to be used for `expect` input in a unit test.
|
|
119
126
|
|
|
120
127
|
Receives:
|
|
121
128
|
manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.
|
|
122
129
|
unit_test (UnitTests): The UnitTests object to check.
|
|
123
130
|
|
|
124
131
|
Other Parameters:
|
|
125
|
-
description (
|
|
126
|
-
exclude (
|
|
127
|
-
include (
|
|
128
|
-
severity (
|
|
132
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
133
|
+
exclude (str | None): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
|
|
134
|
+
include (str | None): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
|
|
135
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
129
136
|
|
|
130
137
|
Example(s):
|
|
131
138
|
```yaml
|
|
@@ -137,21 +144,45 @@ class CheckUnitTestExpectFormats(BaseCheck):
|
|
|
137
144
|
|
|
138
145
|
"""
|
|
139
146
|
|
|
140
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
147
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
141
148
|
name: Literal["check_unit_test_expect_format"]
|
|
142
|
-
permitted_formats:
|
|
149
|
+
permitted_formats: list[Literal["csv", "dict", "sql"]] = Field(
|
|
143
150
|
default=["csv", "dict", "sql"],
|
|
144
151
|
)
|
|
145
|
-
unit_test: "UnitTests" = Field(default=None)
|
|
152
|
+
unit_test: "UnitTests | None" = Field(default=None)
|
|
146
153
|
|
|
147
154
|
def execute(self) -> None:
|
|
148
|
-
"""Execute the check.
|
|
155
|
+
"""Execute the check.
|
|
156
|
+
|
|
157
|
+
Raises:
|
|
158
|
+
DbtBouncerFailedCheckError: If unit test expect format is not permitted.
|
|
159
|
+
|
|
160
|
+
"""
|
|
161
|
+
if self.manifest_obj is None:
|
|
162
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
163
|
+
if self.unit_test is None:
|
|
164
|
+
raise DbtBouncerFailedCheckError("self.unit_test is None")
|
|
149
165
|
if get_package_version_number(
|
|
150
|
-
self.manifest_obj.manifest.metadata.dbt_version
|
|
166
|
+
self.manifest_obj.manifest.metadata.dbt_version or "0.0.0"
|
|
151
167
|
) >= get_package_version_number("1.8.0"):
|
|
152
|
-
|
|
153
|
-
|
|
168
|
+
if self.unit_test.expect.format is None:
|
|
169
|
+
pytest.fail(
|
|
170
|
+
f"Unit test `{self.unit_test.name}` does not have an `expect` format defined. "
|
|
171
|
+
f"Permitted formats are: {self.permitted_formats}."
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
format_value = (
|
|
175
|
+
self.unit_test.expect.format.value
|
|
176
|
+
if self.unit_test.expect.format
|
|
177
|
+
else None
|
|
154
178
|
)
|
|
179
|
+
|
|
180
|
+
if format_value not in self.permitted_formats:
|
|
181
|
+
raise DbtBouncerFailedCheckError(
|
|
182
|
+
f"Unit test `{self.unit_test.name}` has an `expect` format that is not permitted. "
|
|
183
|
+
f"Permitted formats are: {self.permitted_formats}. "
|
|
184
|
+
f"Found: {format_value}"
|
|
185
|
+
)
|
|
155
186
|
else:
|
|
156
187
|
logging.warning(
|
|
157
188
|
"The `check_unit_test_expect_format` check is only supported for dbt 1.8.0 and above.",
|
|
@@ -166,17 +197,17 @@ class CheckUnitTestGivenFormats(BaseCheck):
|
|
|
166
197
|
This check is only supported for dbt 1.8.0 and above.
|
|
167
198
|
|
|
168
199
|
Parameters:
|
|
169
|
-
permitted_formats (
|
|
200
|
+
permitted_formats (list[Literal["csv", "dict", "sql"]] | None): A list of formats that are allowed to be used for `expect` input in a unit test.
|
|
170
201
|
|
|
171
202
|
Receives:
|
|
172
203
|
manifest_obj (DbtBouncerManifest): The DbtBouncerManifest object parsed from `manifest.json`.
|
|
173
204
|
unit_test (UnitTests): The UnitTests object to check.
|
|
174
205
|
|
|
175
206
|
Other Parameters:
|
|
176
|
-
description (
|
|
177
|
-
exclude (
|
|
178
|
-
include (
|
|
179
|
-
severity (
|
|
207
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
208
|
+
exclude (str | None): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Unit test paths that match the pattern will not be checked.
|
|
209
|
+
include (str | None): Regex pattern to match the unit test path (i.e the .yml file where the unit test is configured). Only unit test paths that match the pattern will be checked.
|
|
210
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
180
211
|
|
|
181
212
|
Example(s):
|
|
182
213
|
```yaml
|
|
@@ -188,22 +219,34 @@ class CheckUnitTestGivenFormats(BaseCheck):
|
|
|
188
219
|
|
|
189
220
|
"""
|
|
190
221
|
|
|
191
|
-
manifest_obj: "DbtBouncerManifest" = Field(default=None)
|
|
222
|
+
manifest_obj: "DbtBouncerManifest | None" = Field(default=None)
|
|
192
223
|
name: Literal["check_unit_test_given_formats"]
|
|
193
|
-
permitted_formats:
|
|
224
|
+
permitted_formats: list[Literal["csv", "dict", "sql"]] = Field(
|
|
194
225
|
default=["csv", "dict", "sql"],
|
|
195
226
|
)
|
|
196
|
-
unit_test: "UnitTests" = Field(default=None)
|
|
227
|
+
unit_test: "UnitTests | None" = Field(default=None)
|
|
197
228
|
|
|
198
229
|
def execute(self) -> None:
|
|
199
|
-
"""Execute the check.
|
|
230
|
+
"""Execute the check.
|
|
231
|
+
|
|
232
|
+
Raises:
|
|
233
|
+
DbtBouncerFailedCheckError: If unit test given formats are not permitted.
|
|
234
|
+
|
|
235
|
+
"""
|
|
236
|
+
if self.manifest_obj is None:
|
|
237
|
+
raise DbtBouncerFailedCheckError("self.manifest_obj is None")
|
|
238
|
+
if self.unit_test is None:
|
|
239
|
+
raise DbtBouncerFailedCheckError("self.unit_test is None")
|
|
200
240
|
if get_package_version_number(
|
|
201
|
-
self.manifest_obj.manifest.metadata.dbt_version
|
|
241
|
+
self.manifest_obj.manifest.metadata.dbt_version or "0.0.0"
|
|
202
242
|
) >= get_package_version_number("1.8.0"):
|
|
203
|
-
given_formats = [
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
)
|
|
243
|
+
given_formats = [
|
|
244
|
+
i.format.value for i in self.unit_test.given if i.format is not None
|
|
245
|
+
]
|
|
246
|
+
if not all(e in self.permitted_formats for e in given_formats):
|
|
247
|
+
raise DbtBouncerFailedCheckError(
|
|
248
|
+
f"Unit test `{self.unit_test.name}` has given formats which are not permitted. Permitted formats are: {self.permitted_formats}."
|
|
249
|
+
)
|
|
207
250
|
else:
|
|
208
251
|
logging.warning(
|
|
209
252
|
"The `check_unit_test_given_formats` check is only supported for dbt 1.8.0 and above.",
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING, Literal, Optional
|
|
1
|
+
from typing import TYPE_CHECKING, Literal
|
|
4
2
|
|
|
5
3
|
from pydantic import Field
|
|
6
4
|
|
|
@@ -9,6 +7,8 @@ from dbt_bouncer.check_base import BaseCheck
|
|
|
9
7
|
if TYPE_CHECKING:
|
|
10
8
|
from dbt_bouncer.artifact_parsers.parsers_run_results import DbtBouncerRunResultBase
|
|
11
9
|
|
|
10
|
+
from dbt_bouncer.checks.common import DbtBouncerFailedCheckError
|
|
11
|
+
|
|
12
12
|
|
|
13
13
|
class CheckRunResultsMaxExecutionTime(BaseCheck):
|
|
14
14
|
"""Each result can take a maximum duration (seconds).
|
|
@@ -20,10 +20,10 @@ class CheckRunResultsMaxExecutionTime(BaseCheck):
|
|
|
20
20
|
run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.
|
|
21
21
|
|
|
22
22
|
Other Parameters:
|
|
23
|
-
description (
|
|
24
|
-
exclude (
|
|
25
|
-
include (
|
|
26
|
-
severity (
|
|
23
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
24
|
+
exclude (str | None): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.
|
|
25
|
+
include (str | None): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.
|
|
26
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
27
27
|
|
|
28
28
|
Example(s):
|
|
29
29
|
```yaml
|
|
@@ -42,13 +42,22 @@ class CheckRunResultsMaxExecutionTime(BaseCheck):
|
|
|
42
42
|
|
|
43
43
|
max_execution_time_seconds: float
|
|
44
44
|
name: Literal["check_run_results_max_execution_time"]
|
|
45
|
-
run_result:
|
|
45
|
+
run_result: "DbtBouncerRunResultBase | None" = Field(default=None)
|
|
46
46
|
|
|
47
47
|
def execute(self) -> None:
|
|
48
|
-
"""Execute the check.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
"""Execute the check.
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
DbtBouncerFailedCheckError: If execution time is greater than permitted.
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
if self.run_result is None:
|
|
55
|
+
raise DbtBouncerFailedCheckError("self.run_result is None")
|
|
56
|
+
|
|
57
|
+
if self.run_result.execution_time > self.max_execution_time_seconds:
|
|
58
|
+
raise DbtBouncerFailedCheckError(
|
|
59
|
+
f"`{self.run_result.unique_id.split('.')[-1]}` has an execution time ({self.run_result.execution_time} greater than permitted ({self.max_execution_time_seconds}s)."
|
|
60
|
+
)
|
|
52
61
|
|
|
53
62
|
|
|
54
63
|
class CheckRunResultsMaxGigabytesBilled(BaseCheck):
|
|
@@ -63,10 +72,10 @@ class CheckRunResultsMaxGigabytesBilled(BaseCheck):
|
|
|
63
72
|
run_result (DbtBouncerRunResult): The DbtBouncerRunResult object to check.
|
|
64
73
|
|
|
65
74
|
Other Parameters:
|
|
66
|
-
description (
|
|
67
|
-
exclude (
|
|
68
|
-
include (
|
|
69
|
-
severity (
|
|
75
|
+
description (str | None): Description of what the check does and why it is implemented.
|
|
76
|
+
exclude (str | None): Regex pattern to match the resource path. Resource paths that match the pattern will not be checked.
|
|
77
|
+
include (str | None): Regex pattern to match the resource path. Only resource paths that match the pattern will be checked.
|
|
78
|
+
severity (Literal["error", "warn"] | None): Severity level of the check. Default: `error`.
|
|
70
79
|
|
|
71
80
|
Raises: # noqa:DOC502
|
|
72
81
|
KeyError: If the `dbt-bigquery` adapter is not used.
|
|
@@ -83,15 +92,19 @@ class CheckRunResultsMaxGigabytesBilled(BaseCheck):
|
|
|
83
92
|
|
|
84
93
|
max_gigabytes_billed: float
|
|
85
94
|
name: Literal["check_run_results_max_gigabytes_billed"]
|
|
86
|
-
run_result:
|
|
95
|
+
run_result: "DbtBouncerRunResultBase | None" = Field(default=None)
|
|
87
96
|
|
|
88
97
|
def execute(self) -> None:
|
|
89
98
|
"""Execute the check.
|
|
90
99
|
|
|
91
100
|
Raises:
|
|
101
|
+
DbtBouncerFailedCheckError: If gigabytes billed is greater than permitted.
|
|
92
102
|
RuntimeError: If running with adapter other than `dbt-bigquery`.
|
|
93
103
|
|
|
94
104
|
"""
|
|
105
|
+
if self.run_result is None:
|
|
106
|
+
raise DbtBouncerFailedCheckError("self.run_result is None")
|
|
107
|
+
|
|
95
108
|
try:
|
|
96
109
|
gigabytes_billed = self.run_result.adapter_response["bytes_billed"] / (
|
|
97
110
|
1000**3
|
|
@@ -101,6 +114,7 @@ class CheckRunResultsMaxGigabytesBilled(BaseCheck):
|
|
|
101
114
|
"`bytes_billed` not found in adapter response. Are you using the `dbt-bigquery` adapter?",
|
|
102
115
|
) from e
|
|
103
116
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
117
|
+
if gigabytes_billed > self.max_gigabytes_billed:
|
|
118
|
+
raise DbtBouncerFailedCheckError(
|
|
119
|
+
f"`{self.run_result.unique_id.split('.')[-2]}` results in ({gigabytes_billed} billed bytes, this is greater than permitted ({self.max_gigabytes_billed})."
|
|
120
|
+
)
|
|
@@ -1,46 +1,65 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import operator
|
|
1
3
|
import os
|
|
4
|
+
from functools import reduce
|
|
2
5
|
from pathlib import Path
|
|
3
|
-
from typing import Any,
|
|
6
|
+
from typing import Any, Literal
|
|
4
7
|
|
|
8
|
+
import click
|
|
5
9
|
from pydantic import BaseModel, ConfigDict, Field
|
|
6
10
|
from typing_extensions import Annotated
|
|
7
11
|
|
|
8
|
-
from dbt_bouncer.
|
|
12
|
+
from dbt_bouncer.global_context import get_context
|
|
13
|
+
from dbt_bouncer.utils import clean_path_str, get_check_objects
|
|
9
14
|
|
|
10
15
|
|
|
11
16
|
def get_check_types(
|
|
12
|
-
check_type:
|
|
17
|
+
check_type: list[
|
|
13
18
|
Literal["catalog_checks", "manifest_checks", "run_results_checks"]
|
|
14
19
|
],
|
|
15
|
-
) ->
|
|
20
|
+
) -> list[Any]:
|
|
16
21
|
"""Get the check types from the check categories.
|
|
17
22
|
|
|
18
23
|
Args:
|
|
19
|
-
check_type:
|
|
24
|
+
check_type: list[Literal["catalog_checks", "manifest_checks", "run_results_checks"]]
|
|
20
25
|
|
|
21
26
|
Returns:
|
|
22
|
-
|
|
27
|
+
list[str]: The check types.
|
|
23
28
|
|
|
24
29
|
"""
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
30
|
+
try:
|
|
31
|
+
ctx = get_context()
|
|
32
|
+
if ctx:
|
|
33
|
+
config_file_path = ctx.config_file_path
|
|
34
|
+
custom_checks_dir = ctx.custom_checks_dir
|
|
35
|
+
else:
|
|
36
|
+
click_ctx = click.get_current_context()
|
|
37
|
+
config_file_path = click_ctx.obj.get("config_file_path")
|
|
38
|
+
custom_checks_dir = click_ctx.obj.get("custom_checks_dir")
|
|
39
|
+
|
|
40
|
+
if custom_checks_dir:
|
|
41
|
+
custom_checks_dir = config_file_path.parent / custom_checks_dir
|
|
42
|
+
except (RuntimeError, AttributeError, KeyError):
|
|
43
|
+
custom_checks_dir = None
|
|
44
|
+
|
|
45
|
+
check_classes: list[dict[str, Any | Path]] = [
|
|
28
46
|
{
|
|
29
|
-
"class":
|
|
30
|
-
"source_file": Path(clean_path_str(x
|
|
47
|
+
"class": x,
|
|
48
|
+
"source_file": Path(clean_path_str(inspect.getfile(x))),
|
|
31
49
|
}
|
|
32
|
-
for x in get_check_objects()
|
|
50
|
+
for x in get_check_objects(custom_checks_dir)
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
filtered_classes = [
|
|
54
|
+
x["class"] for x in check_classes if x["source_file"].parts[-2] == check_type
|
|
33
55
|
]
|
|
34
|
-
|
|
56
|
+
if not filtered_classes:
|
|
57
|
+
return list[Any] # type: ignore[return-value]
|
|
58
|
+
|
|
59
|
+
return list[ # type: ignore[misc, return-value]
|
|
35
60
|
Annotated[
|
|
36
61
|
Annotated[
|
|
37
|
-
|
|
38
|
-
tuple(
|
|
39
|
-
x["class"]
|
|
40
|
-
for x in check_classes
|
|
41
|
-
if x["source_file"].parts[-2] == check_type
|
|
42
|
-
)
|
|
43
|
-
],
|
|
62
|
+
reduce(operator.or_, filtered_classes), # type: ignore
|
|
44
63
|
Field(discriminator="name"),
|
|
45
64
|
],
|
|
46
65
|
Field(discriminator="name"),
|
|
@@ -53,29 +72,29 @@ class DbtBouncerConfBase(BaseModel):
|
|
|
53
72
|
|
|
54
73
|
model_config = ConfigDict(extra="forbid")
|
|
55
74
|
|
|
56
|
-
custom_checks_dir:
|
|
75
|
+
custom_checks_dir: str | None = Field(
|
|
57
76
|
default=None,
|
|
58
77
|
description="Path to a directory containing custom checks.",
|
|
59
78
|
)
|
|
60
|
-
dbt_artifacts_dir:
|
|
61
|
-
|
|
79
|
+
dbt_artifacts_dir: str | None = Field(
|
|
80
|
+
default_factory=lambda: (
|
|
62
81
|
f"{os.getenv('DBT_PROJECT_DIR')}/target"
|
|
63
82
|
if os.getenv("DBT_PROJECT_DIR")
|
|
64
83
|
else "./target"
|
|
65
84
|
)
|
|
66
85
|
)
|
|
67
|
-
exclude:
|
|
86
|
+
exclude: str | None = Field(
|
|
68
87
|
default=None,
|
|
69
88
|
description="Regexp to match which paths to exclude.",
|
|
70
89
|
)
|
|
71
|
-
include:
|
|
90
|
+
include: str | None = Field(
|
|
72
91
|
default=None,
|
|
73
92
|
description="Regexp to match which paths to include.",
|
|
74
93
|
)
|
|
75
|
-
package_name:
|
|
94
|
+
package_name: str | None = Field(
|
|
76
95
|
default=None, description="If you want to run `dbt-bouncer` against a package."
|
|
77
96
|
)
|
|
78
|
-
severity:
|
|
97
|
+
severity: Literal["error", "warn"] | None = Field(
|
|
79
98
|
default=None,
|
|
80
99
|
description="Severity of the check, one of 'error' or 'warn'.",
|
|
81
100
|
)
|
|
@@ -90,7 +109,7 @@ class DbtBouncerConfAllCategories(DbtBouncerConfBase):
|
|
|
90
109
|
manifest_checks: get_check_types(check_type="manifest") = Field(default=[]) # type: ignore[valid-type]
|
|
91
110
|
run_results_checks: get_check_types(check_type="run_results") = Field(default=[]) # type: ignore[valid-type]
|
|
92
111
|
|
|
93
|
-
custom_checks_dir:
|
|
112
|
+
custom_checks_dir: str | None = Field(
|
|
94
113
|
default=None,
|
|
95
114
|
description="Path to a directory containing custom checks.",
|
|
96
115
|
)
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
+
from collections.abc import Mapping
|
|
4
5
|
from pathlib import Path, PurePath
|
|
5
|
-
from typing import TYPE_CHECKING, Any,
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
6
7
|
|
|
7
8
|
import click
|
|
8
9
|
import toml
|
|
@@ -29,14 +30,14 @@ DEFAULT_DBT_BOUNCER_CONFIG = """manifest_checks:
|
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def conf_cls_factory(
|
|
32
|
-
check_categories:
|
|
33
|
+
check_categories: list[
|
|
33
34
|
Literal["catalog_checks", "manifest_checks", "run_results_checks"]
|
|
34
35
|
],
|
|
35
36
|
):
|
|
36
37
|
"""Return the appropriate configuration class based on the check categories.
|
|
37
38
|
|
|
38
39
|
Args:
|
|
39
|
-
check_categories:
|
|
40
|
+
check_categories: list[Literal["catalog_checks", "manifest_checks", "run_results_checks"]]
|
|
40
41
|
|
|
41
42
|
Raises:
|
|
42
43
|
ValueError: If the check categories are not valid.
|
|
@@ -144,7 +145,7 @@ def get_config_file_path(
|
|
|
144
145
|
|
|
145
146
|
def load_config_file_contents(
|
|
146
147
|
config_file_path: PurePath,
|
|
147
|
-
allow_default_config_file_creation:
|
|
148
|
+
allow_default_config_file_creation: bool | None = None,
|
|
148
149
|
) -> Mapping[str, Any]:
|
|
149
150
|
"""Load the contents of the config file.
|
|
150
151
|
|
|
@@ -160,7 +161,7 @@ def load_config_file_contents(
|
|
|
160
161
|
|
|
161
162
|
"""
|
|
162
163
|
if config_file_path.suffix in [".yml", ".yaml"]:
|
|
163
|
-
return load_config_from_yaml(config_file_path)
|
|
164
|
+
return load_config_from_yaml(Path(config_file_path))
|
|
164
165
|
elif config_file_path.suffix in [".toml"]:
|
|
165
166
|
toml_cfg = toml.load(config_file_path)
|
|
166
167
|
if "dbt-bouncer" in toml_cfg["tool"]:
|
|
@@ -200,8 +201,8 @@ def load_config_file_contents(
|
|
|
200
201
|
|
|
201
202
|
|
|
202
203
|
def validate_conf(
|
|
203
|
-
check_categories, #:
|
|
204
|
-
config_file_contents:
|
|
204
|
+
check_categories, #: list[Literal["catalog_checks"], Literal["manifest_checks"], Literal["run_results_checks"]],
|
|
205
|
+
config_file_contents: dict[str, Any],
|
|
205
206
|
) -> "DbtBouncerConf":
|
|
206
207
|
"""Validate the configuration and return the Pydantic model.
|
|
207
208
|
|
|
@@ -249,6 +250,8 @@ def validate_conf(
|
|
|
249
250
|
DbtBouncerManifest,
|
|
250
251
|
DbtBouncerModel,
|
|
251
252
|
DbtBouncerModelBase,
|
|
253
|
+
DbtBouncerSeed,
|
|
254
|
+
DbtBouncerSeedBase,
|
|
252
255
|
DbtBouncerSemanticModel,
|
|
253
256
|
DbtBouncerSemanticModelBase,
|
|
254
257
|
DbtBouncerSnapshot,
|
|
@@ -273,7 +276,7 @@ def validate_conf(
|
|
|
273
276
|
except ValidationError as e:
|
|
274
277
|
import jellyfish
|
|
275
278
|
|
|
276
|
-
error_message:
|
|
279
|
+
error_message: list[str] = []
|
|
277
280
|
for error in e.errors():
|
|
278
281
|
if (
|
|
279
282
|
re.compile(
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from pathlib import PurePath
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BouncerContext:
|
|
5
|
+
"""Context object for dbt-bouncer."""
|
|
6
|
+
|
|
7
|
+
def __init__(
|
|
8
|
+
self, config_file_path: PurePath, custom_checks_dir: str | None
|
|
9
|
+
) -> None:
|
|
10
|
+
"""Initialize the context."""
|
|
11
|
+
self.config_file_path = config_file_path
|
|
12
|
+
self.custom_checks_dir = custom_checks_dir
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
_context: BouncerContext | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def set_context(ctx: BouncerContext) -> None:
|
|
19
|
+
"""Set the global context."""
|
|
20
|
+
global _context
|
|
21
|
+
_context = ctx
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_context() -> BouncerContext | None:
|
|
25
|
+
"""Get the global context.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
BouncerContext | None: The global context if set, else None.
|
|
29
|
+
|
|
30
|
+
"""
|
|
31
|
+
return _context
|