dbt-bouncer 1.31.2rc3__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_bouncer/artifact_parsers/dbt_cloud/catalog_latest.py +21 -21
- dbt_bouncer/artifact_parsers/dbt_cloud/manifest_latest.py +1745 -1745
- dbt_bouncer/artifact_parsers/dbt_cloud/run_results_latest.py +22 -22
- dbt_bouncer/artifact_parsers/parsers_catalog.py +26 -24
- dbt_bouncer/artifact_parsers/parsers_common.py +57 -36
- dbt_bouncer/artifact_parsers/parsers_manifest.py +98 -69
- dbt_bouncer/artifact_parsers/parsers_run_results.py +32 -19
- dbt_bouncer/check_base.py +22 -11
- dbt_bouncer/checks/catalog/check_catalog_sources.py +22 -12
- dbt_bouncer/checks/catalog/check_columns.py +175 -105
- dbt_bouncer/checks/common.py +24 -3
- dbt_bouncer/checks/manifest/check_exposures.py +79 -52
- dbt_bouncer/checks/manifest/check_lineage.py +69 -40
- dbt_bouncer/checks/manifest/check_macros.py +177 -104
- dbt_bouncer/checks/manifest/check_metadata.py +28 -18
- dbt_bouncer/checks/manifest/check_models.py +842 -496
- dbt_bouncer/checks/manifest/check_seeds.py +63 -0
- dbt_bouncer/checks/manifest/check_semantic_models.py +28 -20
- dbt_bouncer/checks/manifest/check_snapshots.py +57 -33
- dbt_bouncer/checks/manifest/check_sources.py +246 -137
- dbt_bouncer/checks/manifest/check_unit_tests.py +97 -54
- dbt_bouncer/checks/run_results/check_run_results.py +34 -20
- dbt_bouncer/config_file_parser.py +47 -28
- dbt_bouncer/config_file_validator.py +11 -8
- dbt_bouncer/global_context.py +31 -0
- dbt_bouncer/main.py +128 -67
- dbt_bouncer/runner.py +61 -31
- dbt_bouncer/utils.py +146 -50
- dbt_bouncer/version.py +1 -1
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/METADATA +15 -15
- dbt_bouncer-2.0.0.dist-info/RECORD +37 -0
- dbt_bouncer-1.31.2rc3.dist-info/RECORD +0 -35
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/WHEEL +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/entry_points.txt +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/licenses/LICENSE +0 -0
- {dbt_bouncer-1.31.2rc3.dist-info → dbt_bouncer-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
from enum import Enum
|
|
9
|
-
from typing import Any
|
|
9
|
+
from typing import Any
|
|
10
10
|
|
|
11
11
|
from pydantic import ConfigDict, Field
|
|
12
12
|
|
|
@@ -18,10 +18,10 @@ class Metadata(BaseParserModel):
|
|
|
18
18
|
extra='allow',
|
|
19
19
|
)
|
|
20
20
|
dbt_schema_version: str
|
|
21
|
-
dbt_version:
|
|
22
|
-
generated_at:
|
|
23
|
-
invocation_id:
|
|
24
|
-
env:
|
|
21
|
+
dbt_version: str | None = '1.9.0a1'
|
|
22
|
+
generated_at: str | None = None
|
|
23
|
+
invocation_id: str | None = None
|
|
24
|
+
env: dict[str, str] | None = None
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class Status(Enum):
|
|
@@ -33,7 +33,7 @@ class Status(Enum):
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class Status1(Enum):
|
|
36
|
-
pass_ = 'pass'
|
|
36
|
+
pass_ = 'pass' # nosec
|
|
37
37
|
error = 'error'
|
|
38
38
|
fail = 'fail'
|
|
39
39
|
warn = 'warn'
|
|
@@ -41,7 +41,7 @@ class Status1(Enum):
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
class Status2(Enum):
|
|
44
|
-
pass_ = 'pass'
|
|
44
|
+
pass_ = 'pass' # nosec
|
|
45
45
|
warn = 'warn'
|
|
46
46
|
error = 'error'
|
|
47
47
|
runtime_error = 'runtime error'
|
|
@@ -52,34 +52,34 @@ class TimingItem(BaseParserModel):
|
|
|
52
52
|
extra='allow',
|
|
53
53
|
)
|
|
54
54
|
name: str
|
|
55
|
-
started_at:
|
|
56
|
-
completed_at:
|
|
55
|
+
started_at: str | None = None
|
|
56
|
+
completed_at: str | None = None
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
class BatchResults(BaseParserModel):
|
|
60
60
|
model_config = ConfigDict(
|
|
61
61
|
extra='allow',
|
|
62
62
|
)
|
|
63
|
-
successful:
|
|
64
|
-
failed:
|
|
63
|
+
successful: list[list] | None = None
|
|
64
|
+
failed: list[list] | None = None
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class Result(BaseParserModel):
|
|
68
68
|
model_config = ConfigDict(
|
|
69
69
|
extra='allow',
|
|
70
70
|
)
|
|
71
|
-
status:
|
|
72
|
-
timing:
|
|
71
|
+
status: Status | Status1 | Status2
|
|
72
|
+
timing: list[TimingItem]
|
|
73
73
|
thread_id: str
|
|
74
74
|
execution_time: float
|
|
75
|
-
adapter_response:
|
|
76
|
-
message:
|
|
77
|
-
failures:
|
|
75
|
+
adapter_response: dict[str, Any]
|
|
76
|
+
message: str | None = None
|
|
77
|
+
failures: int | None = None
|
|
78
78
|
unique_id: str
|
|
79
|
-
compiled:
|
|
80
|
-
compiled_code:
|
|
81
|
-
relation_name:
|
|
82
|
-
batch_results:
|
|
79
|
+
compiled: bool | None = None
|
|
80
|
+
compiled_code: str | None = None
|
|
81
|
+
relation_name: str | None = None
|
|
82
|
+
batch_results: BatchResults | None = None
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
class RunResultsLatest(BaseParserModel):
|
|
@@ -87,6 +87,6 @@ class RunResultsLatest(BaseParserModel):
|
|
|
87
87
|
extra='allow',
|
|
88
88
|
)
|
|
89
89
|
metadata: Metadata = Field(..., title='BaseArtifactMetadata')
|
|
90
|
-
results:
|
|
90
|
+
results: list[Result]
|
|
91
91
|
elapsed_time: float
|
|
92
|
-
args:
|
|
92
|
+
args: dict[str, Any] | None = None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import warnings
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel
|
|
6
6
|
|
|
@@ -34,15 +34,15 @@ from dbt_bouncer.artifact_parsers.parsers_common import load_dbt_artifact
|
|
|
34
34
|
class DbtBouncerCatalog(BaseModel):
|
|
35
35
|
"""Model for all catalog objects."""
|
|
36
36
|
|
|
37
|
-
catalog:
|
|
37
|
+
catalog: CatalogV1 | CatalogLatest
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
class DbtBouncerCatalogNode(BaseModel):
|
|
41
41
|
"""Model for all nodes in `catalog.json`."""
|
|
42
42
|
|
|
43
|
-
catalog_node:
|
|
44
|
-
CatalogNodes
|
|
45
|
-
|
|
43
|
+
catalog_node: (
|
|
44
|
+
CatalogNodes | CatalogNodesLatest | CatalogSources | CatalogSourcesLatest
|
|
45
|
+
)
|
|
46
46
|
original_file_path: str
|
|
47
47
|
unique_id: str
|
|
48
48
|
|
|
@@ -50,28 +50,32 @@ class DbtBouncerCatalogNode(BaseModel):
|
|
|
50
50
|
def parse_catalog(
|
|
51
51
|
artifact_dir: "Path",
|
|
52
52
|
manifest_obj: "DbtBouncerManifest",
|
|
53
|
-
package_name:
|
|
54
|
-
) -> tuple[
|
|
53
|
+
package_name: str | None = None,
|
|
54
|
+
) -> tuple[list[DbtBouncerCatalogNode], list[DbtBouncerCatalogNode]]:
|
|
55
55
|
"""Parse the catalog.json artifact.
|
|
56
56
|
|
|
57
57
|
Returns:
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
list[DbtBouncerCatalogNode]: List of catalog nodes for the project.
|
|
59
|
+
list[DbtBouncerCatalogNode]: List of catalog nodes for the project sources.
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
TypeError: If the loaded artifact is not of the expected type.
|
|
60
63
|
|
|
61
64
|
"""
|
|
62
|
-
catalog_obj
|
|
65
|
+
catalog_obj = load_dbt_artifact(
|
|
63
66
|
artifact_name="catalog.json",
|
|
64
67
|
dbt_artifacts_dir=artifact_dir,
|
|
65
68
|
)
|
|
69
|
+
if not isinstance(catalog_obj, DbtBouncerCatalog):
|
|
70
|
+
raise TypeError(f"Expected DbtBouncerCatalog, got {type(catalog_obj)}")
|
|
71
|
+
|
|
66
72
|
project_catalog_nodes = [
|
|
67
73
|
DbtBouncerCatalogNode(
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
"unique_id": k,
|
|
74
|
-
},
|
|
74
|
+
catalog_node=v,
|
|
75
|
+
original_file_path=str(
|
|
76
|
+
clean_path_str(manifest_obj.manifest.nodes[k].original_file_path)
|
|
77
|
+
),
|
|
78
|
+
unique_id=k,
|
|
75
79
|
)
|
|
76
80
|
for k, v in catalog_obj.catalog.nodes.items()
|
|
77
81
|
if k.split(".")[-2]
|
|
@@ -79,13 +83,11 @@ def parse_catalog(
|
|
|
79
83
|
]
|
|
80
84
|
project_catalog_sources = [
|
|
81
85
|
DbtBouncerCatalogNode(
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
"unique_id": k,
|
|
88
|
-
},
|
|
86
|
+
catalog_node=v,
|
|
87
|
+
original_file_path=str(
|
|
88
|
+
clean_path_str(manifest_obj.manifest.sources[k].original_file_path)
|
|
89
|
+
),
|
|
90
|
+
unique_id=k,
|
|
89
91
|
)
|
|
90
92
|
for k, v in catalog_obj.catalog.sources.items()
|
|
91
93
|
if k.split(".")[1]
|
|
@@ -1,19 +1,26 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from typing import TYPE_CHECKING,
|
|
4
|
+
from typing import TYPE_CHECKING, Literal
|
|
5
5
|
|
|
6
6
|
from dbt_bouncer.utils import get_package_version_number
|
|
7
7
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
|
-
from
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
from dbt_artifacts_parser.parsers.run_results.run_results_v5 import RunResultsV5
|
|
10
|
+
|
|
11
|
+
from dbt_bouncer.artifact_parsers.dbt_cloud.run_results_latest import (
|
|
12
|
+
RunResultsLatest,
|
|
13
|
+
)
|
|
14
|
+
from dbt_bouncer.artifact_parsers.parsers_catalog import (
|
|
15
|
+
DbtBouncerCatalog,
|
|
16
|
+
DbtBouncerCatalogNode,
|
|
12
17
|
)
|
|
13
|
-
from dbt_bouncer.artifact_parsers.parsers_catalog import DbtBouncerCatalogNode
|
|
14
18
|
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
19
|
+
DbtBouncerExposureBase,
|
|
20
|
+
DbtBouncerMacroBase,
|
|
15
21
|
DbtBouncerManifest,
|
|
16
22
|
DbtBouncerModel,
|
|
23
|
+
DbtBouncerSeed,
|
|
17
24
|
DbtBouncerSemanticModel,
|
|
18
25
|
DbtBouncerSnapshot,
|
|
19
26
|
DbtBouncerSource,
|
|
@@ -22,7 +29,6 @@ if TYPE_CHECKING:
|
|
|
22
29
|
)
|
|
23
30
|
from dbt_bouncer.artifact_parsers.parsers_run_results import (
|
|
24
31
|
DbtBouncerRunResult,
|
|
25
|
-
DbtBouncerRunResultBase,
|
|
26
32
|
)
|
|
27
33
|
from dbt_bouncer.config_file_parser import (
|
|
28
34
|
DbtBouncerConfAllCategories as DbtBouncerConf,
|
|
@@ -32,14 +38,16 @@ if TYPE_CHECKING:
|
|
|
32
38
|
def load_dbt_artifact(
|
|
33
39
|
artifact_name: Literal["catalog.json", "manifest.json", "run_results.json"],
|
|
34
40
|
dbt_artifacts_dir: Path,
|
|
35
|
-
) ->
|
|
41
|
+
) -> "DbtBouncerCatalog | DbtBouncerManifest | RunResultsV5 | RunResultsLatest":
|
|
36
42
|
"""Load a dbt artifact from a JSON file to a Pydantic object.
|
|
37
43
|
|
|
38
44
|
Returns:
|
|
39
|
-
|
|
45
|
+
DbtBouncerCatalog | DbtBouncerManifest | RunResultsV5 | RunResultsLatest:
|
|
40
46
|
The dbt artifact loaded as a Pydantic object.
|
|
41
47
|
|
|
42
48
|
Raises:
|
|
49
|
+
AssertionError:
|
|
50
|
+
If the dbt version is below the minimum supported version of 1.7.0.
|
|
43
51
|
FileNotFoundError:
|
|
44
52
|
If the artifact file does not exist.
|
|
45
53
|
|
|
@@ -67,11 +75,12 @@ def load_dbt_artifact(
|
|
|
67
75
|
with Path.open(Path(artifact_path), "r") as fp:
|
|
68
76
|
manifest_json = json.load(fp)
|
|
69
77
|
|
|
70
|
-
|
|
78
|
+
if not get_package_version_number(
|
|
71
79
|
manifest_json["metadata"]["dbt_version"]
|
|
72
|
-
) >= get_package_version_number("1.7.0")
|
|
73
|
-
|
|
74
|
-
|
|
80
|
+
) >= get_package_version_number("1.7.0"):
|
|
81
|
+
raise AssertionError(
|
|
82
|
+
f"The supplied `manifest.json` was generated with dbt version {manifest_json['metadata']['dbt_version']}, this is below the minimum supported version of 1.7.0."
|
|
83
|
+
)
|
|
75
84
|
|
|
76
85
|
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
77
86
|
DbtBouncerManifest,
|
|
@@ -94,17 +103,18 @@ def parse_dbt_artifacts(
|
|
|
94
103
|
bouncer_config: "DbtBouncerConf", dbt_artifacts_dir: Path
|
|
95
104
|
) -> tuple[
|
|
96
105
|
"DbtBouncerManifest",
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
106
|
+
list["DbtBouncerExposureBase"],
|
|
107
|
+
list["DbtBouncerMacroBase"],
|
|
108
|
+
list["DbtBouncerModel"],
|
|
109
|
+
list["DbtBouncerSeed"],
|
|
110
|
+
list["DbtBouncerSemanticModel"],
|
|
111
|
+
list["DbtBouncerSnapshot"],
|
|
112
|
+
list["DbtBouncerSource"],
|
|
113
|
+
list["DbtBouncerTest"],
|
|
114
|
+
list["UnitTests"],
|
|
115
|
+
list["DbtBouncerCatalogNode"],
|
|
116
|
+
list["DbtBouncerCatalogNode"],
|
|
117
|
+
list["DbtBouncerRunResult"],
|
|
108
118
|
]:
|
|
109
119
|
"""Parse all required dbt artifacts.
|
|
110
120
|
|
|
@@ -114,31 +124,41 @@ def parse_dbt_artifacts(
|
|
|
114
124
|
|
|
115
125
|
Returns:
|
|
116
126
|
DbtBouncerManifest: The manifest object.
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
127
|
+
list[DbtBouncerExposure]: List of exposures in the project.
|
|
128
|
+
list[DbtBouncerMacro]: List of macros in the project.
|
|
129
|
+
list[DbtBouncerModel]: List of models in the project.
|
|
130
|
+
list[DbtBouncerSeed]: List of seeds in the project.
|
|
131
|
+
list[DbtBouncerSemanticModel]: List of semantic models in the project.
|
|
132
|
+
list[DbtBouncerSnapshot]: List of snapshots in the project.
|
|
133
|
+
list[DbtBouncerSource]: List of sources in the project.
|
|
134
|
+
list[DbtBouncerTest]: List of tests in the project.
|
|
135
|
+
list[DbtBouncerUnitTest]: List of unit tests in the project.
|
|
136
|
+
list[DbtBouncerCatalogNode]: List of catalog nodes for the project.
|
|
137
|
+
list[DbtBouncerCatalogNode]: List of catalog nodes for the project sources.
|
|
138
|
+
list[DbtBouncerRunResult]: A list of DbtBouncerRunResult objects.
|
|
139
|
+
|
|
140
|
+
Raises:
|
|
141
|
+
TypeError: If any of the loaded artifacts are not of the expected type.
|
|
128
142
|
|
|
129
143
|
"""
|
|
130
|
-
from dbt_bouncer.artifact_parsers.
|
|
131
|
-
|
|
144
|
+
from dbt_bouncer.artifact_parsers.parsers_manifest import (
|
|
145
|
+
DbtBouncerManifest,
|
|
146
|
+
parse_manifest_artifact,
|
|
147
|
+
)
|
|
132
148
|
|
|
133
149
|
# Manifest, will always be parsed
|
|
134
150
|
manifest_obj = load_dbt_artifact(
|
|
135
151
|
artifact_name="manifest.json",
|
|
136
152
|
dbt_artifacts_dir=dbt_artifacts_dir,
|
|
137
153
|
)
|
|
154
|
+
if not isinstance(manifest_obj, DbtBouncerManifest):
|
|
155
|
+
raise TypeError(f"Expected DbtBouncerManifest, got {type(manifest_obj)}")
|
|
156
|
+
|
|
138
157
|
(
|
|
139
158
|
project_exposures,
|
|
140
159
|
project_macros,
|
|
141
160
|
project_models,
|
|
161
|
+
project_seeds,
|
|
142
162
|
project_semantic_models,
|
|
143
163
|
project_snapshots,
|
|
144
164
|
project_sources,
|
|
@@ -186,6 +206,7 @@ def parse_dbt_artifacts(
|
|
|
186
206
|
project_exposures,
|
|
187
207
|
project_macros,
|
|
188
208
|
project_models,
|
|
209
|
+
project_seeds,
|
|
189
210
|
project_semantic_models,
|
|
190
211
|
project_snapshots,
|
|
191
212
|
project_sources,
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import warnings
|
|
3
3
|
from enum import Enum
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, TypeAlias, cast
|
|
5
5
|
|
|
6
6
|
from pydantic import BaseModel
|
|
7
7
|
|
|
8
8
|
from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import ManifestLatest
|
|
9
|
+
from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
|
|
10
|
+
Nodes as SeedsLatest,
|
|
11
|
+
)
|
|
9
12
|
from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
|
|
10
13
|
Nodes2 as Nodes2Latest,
|
|
11
14
|
)
|
|
@@ -31,10 +34,16 @@ with warnings.catch_warnings():
|
|
|
31
34
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import (
|
|
32
35
|
GenericTestNode as GenericTestNode_v11,
|
|
33
36
|
)
|
|
37
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import (
|
|
38
|
+
Macro as Macro_v11,
|
|
39
|
+
)
|
|
34
40
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
|
|
35
41
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import (
|
|
36
42
|
ModelNode as ModelNode_v11,
|
|
37
43
|
)
|
|
44
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import (
|
|
45
|
+
SeedNode as SeedNode_v11,
|
|
46
|
+
)
|
|
38
47
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import (
|
|
39
48
|
SemanticModel as SemanticModel_v11,
|
|
40
49
|
)
|
|
@@ -65,10 +74,10 @@ from dbt_bouncer.artifact_parsers.dbt_cloud.manifest_latest import (
|
|
|
65
74
|
class DbtBouncerManifest(BaseModel):
|
|
66
75
|
"""Model for all manifest objects."""
|
|
67
76
|
|
|
68
|
-
manifest:
|
|
77
|
+
manifest: ManifestV11 | ManifestLatest
|
|
69
78
|
|
|
70
79
|
|
|
71
|
-
DbtBouncerExposureBase =
|
|
80
|
+
DbtBouncerExposureBase: TypeAlias = Exposure_v11 | Exposures
|
|
72
81
|
|
|
73
82
|
|
|
74
83
|
class DbtBouncerExposure(BaseModel):
|
|
@@ -79,7 +88,10 @@ class DbtBouncerExposure(BaseModel):
|
|
|
79
88
|
unique_id: str
|
|
80
89
|
|
|
81
90
|
|
|
82
|
-
|
|
91
|
+
DbtBouncerMacroBase: TypeAlias = Macro_v11 | Macros
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
DbtBouncerModelBase: TypeAlias = ModelNode_v11 | Nodes4 | Nodes4Latest
|
|
83
95
|
|
|
84
96
|
|
|
85
97
|
class DbtBouncerModel(BaseModel):
|
|
@@ -90,9 +102,20 @@ class DbtBouncerModel(BaseModel):
|
|
|
90
102
|
unique_id: str
|
|
91
103
|
|
|
92
104
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
105
|
+
DbtBouncerSeedBase: TypeAlias = SeedNode_v11 | SeedsLatest
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class DbtBouncerSeed(BaseModel):
|
|
109
|
+
"""Model for all seed nodes in `manifest.json`."""
|
|
110
|
+
|
|
111
|
+
original_file_path: str
|
|
112
|
+
seed: DbtBouncerSeedBase
|
|
113
|
+
unique_id: str
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
DbtBouncerSemanticModelBase: TypeAlias = (
|
|
117
|
+
SemanticModel_v11 | SemanticModels | SemanticModelsLatest
|
|
118
|
+
)
|
|
96
119
|
|
|
97
120
|
|
|
98
121
|
class DbtBouncerSemanticModel(BaseModel):
|
|
@@ -103,7 +126,7 @@ class DbtBouncerSemanticModel(BaseModel):
|
|
|
103
126
|
unique_id: str
|
|
104
127
|
|
|
105
128
|
|
|
106
|
-
DbtBouncerSnapshotBase =
|
|
129
|
+
DbtBouncerSnapshotBase: TypeAlias = Nodes7 | SnapshotNode_v11
|
|
107
130
|
|
|
108
131
|
|
|
109
132
|
class DbtBouncerSnapshot(BaseModel):
|
|
@@ -114,7 +137,7 @@ class DbtBouncerSnapshot(BaseModel):
|
|
|
114
137
|
unique_id: str
|
|
115
138
|
|
|
116
139
|
|
|
117
|
-
DbtBouncerSourceBase =
|
|
140
|
+
DbtBouncerSourceBase: TypeAlias = SourceDefinition_v11 | Sources | SourcesLatest
|
|
118
141
|
|
|
119
142
|
|
|
120
143
|
class DbtBouncerSource(BaseModel):
|
|
@@ -125,15 +148,15 @@ class DbtBouncerSource(BaseModel):
|
|
|
125
148
|
unique_id: str
|
|
126
149
|
|
|
127
150
|
|
|
128
|
-
DbtBouncerTestBase =
|
|
129
|
-
GenericTestNode_v11
|
|
130
|
-
SingularTestNode_v11
|
|
131
|
-
Nodes1
|
|
132
|
-
Nodes2
|
|
133
|
-
Nodes6
|
|
134
|
-
Nodes2Latest
|
|
135
|
-
Nodes6Latest
|
|
136
|
-
|
|
151
|
+
DbtBouncerTestBase: TypeAlias = (
|
|
152
|
+
GenericTestNode_v11
|
|
153
|
+
| SingularTestNode_v11
|
|
154
|
+
| Nodes1
|
|
155
|
+
| Nodes2
|
|
156
|
+
| Nodes6
|
|
157
|
+
| Nodes2Latest
|
|
158
|
+
| Nodes6Latest
|
|
159
|
+
)
|
|
137
160
|
|
|
138
161
|
|
|
139
162
|
class DbtBouncerTest(BaseModel):
|
|
@@ -145,8 +168,8 @@ class DbtBouncerTest(BaseModel):
|
|
|
145
168
|
|
|
146
169
|
|
|
147
170
|
def parse_manifest(
|
|
148
|
-
manifest:
|
|
149
|
-
) ->
|
|
171
|
+
manifest: dict[str, Any],
|
|
172
|
+
) -> ManifestV11 | ManifestLatest:
|
|
150
173
|
"""Parse manifest.json.
|
|
151
174
|
|
|
152
175
|
Args:
|
|
@@ -156,7 +179,7 @@ def parse_manifest(
|
|
|
156
179
|
ValueError: If the manifest.json is not a valid manifest.json
|
|
157
180
|
|
|
158
181
|
Returns:
|
|
159
|
-
|
|
182
|
+
ManifestV11 | ManifestLatest
|
|
160
183
|
|
|
161
184
|
"""
|
|
162
185
|
dbt_schema_version = manifest["metadata"]["dbt_schema_version"]
|
|
@@ -173,28 +196,30 @@ def parse_manifest(
|
|
|
173
196
|
|
|
174
197
|
|
|
175
198
|
def parse_manifest_artifact(
|
|
176
|
-
manifest_obj: DbtBouncerManifest, package_name:
|
|
199
|
+
manifest_obj: DbtBouncerManifest, package_name: str | None = None
|
|
177
200
|
) -> tuple[
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
201
|
+
list[DbtBouncerExposureBase],
|
|
202
|
+
list[DbtBouncerMacroBase],
|
|
203
|
+
list[DbtBouncerModel],
|
|
204
|
+
list[DbtBouncerSeed],
|
|
205
|
+
list[DbtBouncerSemanticModel],
|
|
206
|
+
list[DbtBouncerSnapshot],
|
|
207
|
+
list[DbtBouncerSource],
|
|
208
|
+
list[DbtBouncerTest],
|
|
209
|
+
list[UnitTests],
|
|
186
210
|
]:
|
|
187
211
|
"""Parse the manifest.json artifact.
|
|
188
212
|
|
|
189
213
|
Returns:
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
214
|
+
list[DbtBouncerExposure]: List of exposures in the project.
|
|
215
|
+
list[DbtBouncerMacro]: List of macros in the project.
|
|
216
|
+
list[DbtBouncerModel]: List of models in the project.
|
|
217
|
+
list[DbtBouncerSeed]: List of seeds in the project.
|
|
218
|
+
list[DbtBouncerSemanticModel]: List of semantic models in the project.
|
|
219
|
+
list[DbtBouncerSnapshot]: List of snapshots in the project.
|
|
220
|
+
list[DbtBouncerSource]: List of sources in the project.
|
|
221
|
+
list[DbtBouncerTest]: List of tests in the project.
|
|
222
|
+
list[DbtBouncerUnitTest]: List of unit tests in the project.
|
|
198
223
|
|
|
199
224
|
"""
|
|
200
225
|
project_exposures = [
|
|
@@ -210,6 +235,7 @@ def parse_manifest_artifact(
|
|
|
210
235
|
== (package_name or manifest_obj.manifest.metadata.project_name)
|
|
211
236
|
]
|
|
212
237
|
project_models = []
|
|
238
|
+
project_seeds = []
|
|
213
239
|
project_snapshots = []
|
|
214
240
|
project_tests = []
|
|
215
241
|
for k, v in manifest_obj.manifest.nodes.items():
|
|
@@ -221,14 +247,24 @@ def parse_manifest_artifact(
|
|
|
221
247
|
):
|
|
222
248
|
project_models.append(
|
|
223
249
|
DbtBouncerModel(
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
"original_file_path": clean_path_str(v.original_file_path),
|
|
228
|
-
"unique_id": k,
|
|
229
|
-
},
|
|
250
|
+
model=cast("Any", v),
|
|
251
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
252
|
+
unique_id=k,
|
|
230
253
|
),
|
|
231
254
|
)
|
|
255
|
+
elif (
|
|
256
|
+
(isinstance(v.resource_type, Enum) and v.resource_type.value == "seed")
|
|
257
|
+
or v.resource_type == "seed"
|
|
258
|
+
) and v.package_name == (
|
|
259
|
+
package_name or manifest_obj.manifest.metadata.project_name
|
|
260
|
+
):
|
|
261
|
+
project_seeds.append(
|
|
262
|
+
DbtBouncerSeed(
|
|
263
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
264
|
+
seed=cast("Any", v),
|
|
265
|
+
unique_id=k,
|
|
266
|
+
),
|
|
267
|
+
)
|
|
232
268
|
elif (
|
|
233
269
|
(isinstance(v.resource_type, Enum) and v.resource_type.value == "snapshot")
|
|
234
270
|
or v.resource_type == "snapshot"
|
|
@@ -237,11 +273,9 @@ def parse_manifest_artifact(
|
|
|
237
273
|
):
|
|
238
274
|
project_snapshots.append(
|
|
239
275
|
DbtBouncerSnapshot(
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
"unique_id": k,
|
|
244
|
-
},
|
|
276
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
277
|
+
snapshot=cast("Any", v),
|
|
278
|
+
unique_id=k,
|
|
245
279
|
),
|
|
246
280
|
)
|
|
247
281
|
elif (
|
|
@@ -252,20 +286,18 @@ def parse_manifest_artifact(
|
|
|
252
286
|
):
|
|
253
287
|
project_tests.append(
|
|
254
288
|
DbtBouncerTest(
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
"unique_id": k,
|
|
259
|
-
},
|
|
289
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
290
|
+
test=cast("Any", v),
|
|
291
|
+
unique_id=k,
|
|
260
292
|
),
|
|
261
293
|
)
|
|
262
294
|
|
|
263
295
|
if get_package_version_number(
|
|
264
|
-
manifest_obj.manifest.metadata.dbt_version
|
|
296
|
+
manifest_obj.manifest.metadata.dbt_version or "0.0.0"
|
|
265
297
|
) >= get_package_version_number("1.8.0"):
|
|
266
298
|
project_unit_tests = [
|
|
267
299
|
v
|
|
268
|
-
for _, v in manifest_obj.manifest
|
|
300
|
+
for _, v in getattr(manifest_obj.manifest, "unit_tests", {}).items()
|
|
269
301
|
if v.package_name
|
|
270
302
|
== (package_name or manifest_obj.manifest.metadata.project_name)
|
|
271
303
|
]
|
|
@@ -274,37 +306,34 @@ def parse_manifest_artifact(
|
|
|
274
306
|
|
|
275
307
|
project_semantic_models = [
|
|
276
308
|
DbtBouncerSemanticModel(
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
"unique_id": k,
|
|
281
|
-
},
|
|
309
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
310
|
+
semantic_model=cast("Any", v),
|
|
311
|
+
unique_id=k,
|
|
282
312
|
)
|
|
283
|
-
for
|
|
313
|
+
for k, v in manifest_obj.manifest.semantic_models.items()
|
|
284
314
|
if v.package_name
|
|
285
315
|
== (package_name or manifest_obj.manifest.metadata.project_name)
|
|
286
316
|
]
|
|
287
317
|
|
|
288
318
|
project_sources = [
|
|
289
319
|
DbtBouncerSource(
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
"unique_id": k,
|
|
294
|
-
},
|
|
320
|
+
original_file_path=str(clean_path_str(v.original_file_path)),
|
|
321
|
+
source=cast("Any", v),
|
|
322
|
+
unique_id=k,
|
|
295
323
|
)
|
|
296
|
-
for
|
|
324
|
+
for k, v in manifest_obj.manifest.sources.items()
|
|
297
325
|
if v.package_name
|
|
298
326
|
== (package_name or manifest_obj.manifest.metadata.project_name)
|
|
299
327
|
]
|
|
300
328
|
|
|
301
329
|
logging.info(
|
|
302
|
-
f"Parsed `manifest.json`, found `{(package_name or manifest_obj.manifest.metadata.project_name)}` project: {len(project_exposures)} exposures, {len(project_macros)} macros, {len(project_models)} nodes, {len(project_semantic_models)} semantic models, {len(project_snapshots)} snapshots, {len(project_sources)} sources, {len(project_tests)} tests, {len(project_unit_tests)} unit tests.",
|
|
330
|
+
f"Parsed `manifest.json`, found `{(package_name or manifest_obj.manifest.metadata.project_name)}` project: {len(project_exposures)} exposures, {len(project_macros)} macros, {len(project_models)} nodes, {len(project_seeds)} seeds, {len(project_semantic_models)} semantic models, {len(project_snapshots)} snapshots, {len(project_sources)} sources, {len(project_tests)} tests, {len(project_unit_tests)} unit tests.",
|
|
303
331
|
)
|
|
304
332
|
return (
|
|
305
333
|
project_exposures,
|
|
306
334
|
project_macros,
|
|
307
335
|
project_models,
|
|
336
|
+
project_seeds,
|
|
308
337
|
project_semantic_models,
|
|
309
338
|
project_snapshots,
|
|
310
339
|
project_sources,
|