altimate-datapilot-cli 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/METADATA +7 -6
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/RECORD +26 -20
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/WHEEL +1 -1
- datapilot/__init__.py +1 -1
- datapilot/clients/altimate/client.py +18 -1
- datapilot/clients/altimate/utils.py +30 -0
- datapilot/core/insights/sql/base/insight.py +2 -9
- datapilot/core/platforms/dbt/cli/cli.py +21 -2
- datapilot/core/platforms/dbt/constants.py +2 -0
- datapilot/core/platforms/dbt/executor.py +69 -0
- datapilot/core/platforms/dbt/factory.py +4 -0
- datapilot/core/platforms/dbt/insights/__init__.py +2 -0
- datapilot/core/platforms/dbt/insights/base.py +3 -0
- datapilot/core/platforms/dbt/insights/sql/__init__.py +0 -0
- datapilot/core/platforms/dbt/insights/sql/base.py +23 -0
- datapilot/core/platforms/dbt/insights/sql/sql_check.py +101 -0
- datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +5 -0
- datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +5 -0
- datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py +0 -0
- datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py +36 -0
- datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py +414 -0
- datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py +5 -0
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/AUTHORS.rst +0 -0
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/LICENSE +0 -0
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/entry_points.txt +0 -0
- {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/top_level.txt +0 -0
{altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: altimate-datapilot-cli
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.14
|
4
4
|
Summary: Assistant for Data Teams
|
5
5
|
Home-page: https://github.com/AltimateAI/datapilot-cli
|
6
6
|
Author: Altimate Inc
|
@@ -30,11 +30,12 @@ Classifier: Topic :: Utilities
|
|
30
30
|
Requires-Python: >=3.8
|
31
31
|
License-File: LICENSE
|
32
32
|
License-File: AUTHORS.rst
|
33
|
-
Requires-Dist: click
|
34
|
-
Requires-Dist: dbt-artifacts-parser
|
35
|
-
Requires-Dist: ruamel.yaml
|
36
|
-
Requires-Dist: tabulate
|
37
|
-
Requires-Dist: requests
|
33
|
+
Requires-Dist: click==8.1.7
|
34
|
+
Requires-Dist: dbt-artifacts-parser==0.6.0
|
35
|
+
Requires-Dist: ruamel.yaml==0.18.6
|
36
|
+
Requires-Dist: tabulate==0.9.0
|
37
|
+
Requires-Dist: requests==2.31.0
|
38
|
+
Requires-Dist: sqlglot==25.30.0
|
38
39
|
|
39
40
|
========
|
40
41
|
Overview
|
@@ -1,11 +1,11 @@
|
|
1
|
-
datapilot/__init__.py,sha256=
|
1
|
+
datapilot/__init__.py,sha256=71Tw8stu19MsLPyF1q_DF5mb_BefzOTPYggLlSLvCms,23
|
2
2
|
datapilot/__main__.py,sha256=I9USmeNnK-cAHb6LZfydJC0LeNSE8enieeY55wpR6uw,380
|
3
3
|
datapilot/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
4
|
datapilot/cli/main.py,sha256=VSdqlkCiu8GSG9qQh8q0BzyocsQc4lKWxZAPEsjXF18,181
|
5
5
|
datapilot/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
6
|
datapilot/clients/altimate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
-
datapilot/clients/altimate/client.py,sha256=
|
8
|
-
datapilot/clients/altimate/utils.py,sha256=
|
7
|
+
datapilot/clients/altimate/client.py,sha256=lLIb9wu--5Rk3qpgnmqeyHlWiPmiTIE-nyMXdWodQ5I,3875
|
8
|
+
datapilot/clients/altimate/utils.py,sha256=B94GQgaiBbw4GrLrK8P7r89zzVeYIzr9-8tuJhoN2MY,4302
|
9
9
|
datapilot/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
10
|
datapilot/config/config.py,sha256=kyj53Qsb85V4iGQsX0vSwULOjscMOSFrJDJ3tnagJpo,403
|
11
11
|
datapilot/config/utils.py,sha256=DIAVX-OZ5Lc0Ky_A7dvdbPcD1QSg2DRxZcuaIIZ2rhw,1146
|
@@ -18,23 +18,23 @@ datapilot/core/insights/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
|
|
18
18
|
datapilot/core/insights/base/insight.py,sha256=AshjeLhV7cxwYANfSidDfHZssifTmvymkGHcWkeKipk,883
|
19
19
|
datapilot/core/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
20
|
datapilot/core/insights/sql/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
21
|
-
datapilot/core/insights/sql/base/insight.py,sha256=
|
21
|
+
datapilot/core/insights/sql/base/insight.py,sha256=k8UUn0qrN-QG6NCunPl7Hd6L6kd1X1eUAeGEsyl8v0o,250
|
22
22
|
datapilot/core/insights/sql/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
23
|
datapilot/core/insights/sql/static/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
24
24
|
datapilot/core/platforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
25
25
|
datapilot/core/platforms/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
-
datapilot/core/platforms/dbt/constants.py,sha256=
|
26
|
+
datapilot/core/platforms/dbt/constants.py,sha256=kW4PEsYWosfDjBEZ9JUhWFtMxKMSNoIn0DNPlJEJuYc,515
|
27
27
|
datapilot/core/platforms/dbt/exceptions.py,sha256=IC5BgcU90gjYYwPcfTlPNtn0_p8fYjavDRMpKZQ0OnY,110
|
28
|
-
datapilot/core/platforms/dbt/executor.py,sha256=
|
29
|
-
datapilot/core/platforms/dbt/factory.py,sha256=
|
28
|
+
datapilot/core/platforms/dbt/executor.py,sha256=4bRNg6yxU1rHu5MDGdq1vO-A0EKS38MNP1hAAkO5h6Y,9992
|
29
|
+
datapilot/core/platforms/dbt/factory.py,sha256=YIQtb-FQQAJsifJ3KiLjjk0WIKTHtEPTNu2MeKHdMG8,1590
|
30
30
|
datapilot/core/platforms/dbt/formatting.py,sha256=bpfa7XmVghTq4WnGDGYC6DruwOwH8YmjFHghoo5cPD8,1638
|
31
31
|
datapilot/core/platforms/dbt/utils.py,sha256=ozFHprR6LTLXQdrGyaRoyIBTua4P1NkP8T7LGgN-9c0,18577
|
32
32
|
datapilot/core/platforms/dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
-
datapilot/core/platforms/dbt/cli/cli.py,sha256=
|
33
|
+
datapilot/core/platforms/dbt/cli/cli.py,sha256=1phfriG4f5pS1UFwBcZ0uM49iovVpnjdama9lSYcKMU,6266
|
34
34
|
datapilot/core/platforms/dbt/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
35
|
datapilot/core/platforms/dbt/hooks/executor_hook.py,sha256=gSM50vAO7C-f1rdnHogWbqc87aCXPXysZepjp5L2qzw,2966
|
36
|
-
datapilot/core/platforms/dbt/insights/__init__.py,sha256=
|
37
|
-
datapilot/core/platforms/dbt/insights/base.py,sha256=
|
36
|
+
datapilot/core/platforms/dbt/insights/__init__.py,sha256=hk7BAzCTDkY8WNV6L0v-CPn9mrsDyJJusoQxNxGyzAY,7634
|
37
|
+
datapilot/core/platforms/dbt/insights/base.py,sha256=WBp3knDE2GJx-aM-gHhOWWSu1mdAJHzxLd243sLRdRw,5366
|
38
38
|
datapilot/core/platforms/dbt/insights/schema.py,sha256=4nPxEGsgN5sCXQ1BrOesTRphkMKZj1szyk6ckoz49eg,2657
|
39
39
|
datapilot/core/platforms/dbt/insights/utils.py,sha256=2btHwxzxEm7cjoLSzzbnZ3q3caI-58NJeWM0_TouJhI,267
|
40
40
|
datapilot/core/platforms/dbt/insights/checks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -99,6 +99,9 @@ datapilot/core/platforms/dbt/insights/performance/__init__.py,sha256=47DEQpj8HBS
|
|
99
99
|
datapilot/core/platforms/dbt/insights/performance/base.py,sha256=4EE7WmwlqMQSmBcpYeZrellvGU87F81cTL5wxBV5QSo,829
|
100
100
|
datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py,sha256=gqL4qqNAybOTYhc9VGlrbQQJ1PkDtBHFKrcQi_XRpTM,3627
|
101
101
|
datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py,sha256=bX7yqls9eMHShXeNdZ8J3IefM3Gp-4D9OGXAYfrKHEs,4689
|
102
|
+
datapilot/core/platforms/dbt/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
103
|
+
datapilot/core/platforms/dbt/insights/sql/base.py,sha256=dpmHtX3rg1_ewECcsdiVkndftK0EBOnvOfsLw2TUg8U,733
|
104
|
+
datapilot/core/platforms/dbt/insights/sql/sql_check.py,sha256=a84YLtjJFtF1NRBbWueSdAwp6NNi6ZU8_ScbwLfpnfk,4652
|
102
105
|
datapilot/core/platforms/dbt/insights/structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
103
106
|
datapilot/core/platforms/dbt/insights/structure/base.py,sha256=sekRrLxB8IvpCzxShPC1L71K36XjsRMJyXzGRz4TcGQ,1101
|
104
107
|
datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py,sha256=NdsXojyRpVv0-jhUWdxg6uvEa0EInT_RR8DYaAsRXKQ,4382
|
@@ -114,13 +117,16 @@ datapilot/core/platforms/dbt/wrappers/catalog/wrapper.py,sha256=LDjt4pSiqGAmvI5D
|
|
114
117
|
datapilot/core/platforms/dbt/wrappers/catalog/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
115
118
|
datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py,sha256=OHCxG0CJAxOgicGeykoJJ3AnJG8izZvDUGNm7pXajRQ,832
|
116
119
|
datapilot/core/platforms/dbt/wrappers/manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
117
|
-
datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=
|
120
|
+
datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=Q9DG-OEyJAa4ePoVhxG78pehfNotNc9A3xtz171fLhU,1189
|
118
121
|
datapilot/core/platforms/dbt/wrappers/manifest/v10/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
119
122
|
datapilot/core/platforms/dbt/wrappers/manifest/v10/schemas.py,sha256=VxqIFQbAL88ObQW1E8wPpiuooCdawdACJUFhgAYBxmo,1523
|
120
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=
|
123
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=7ewyoNPQbjcyNH90cugZh9Nad-cM3bF7XvlZKZ1uEGE,17536
|
121
124
|
datapilot/core/platforms/dbt/wrappers/manifest/v11/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
122
125
|
datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py,sha256=Mb0N48czcwQTz5mxQx2QlVINzz50A5FUm0kMJ-Nzs6A,1523
|
123
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=
|
126
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=DXOxuRHphwKfAr3WPoULteeZGdF8IHlHQu2lm382hrs,17536
|
127
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
128
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py,sha256=Muu2CVkTKbOtPrOcitysh6bxDrdLVw-Cu6HZT8J7Si8,1248
|
129
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py,sha256=ONARXJ3MhttSNUGnIc73CAmSiE2pZctHztwKLmZUGqg,17767
|
124
130
|
datapilot/core/platforms/dbt/wrappers/run_results/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
125
131
|
datapilot/core/platforms/dbt/wrappers/run_results/run_results.py,sha256=3E_y1gAF491WmXt-Z_Fqhr5BU-kVnzjHpZZv5UpOx-s,1267
|
126
132
|
datapilot/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -133,10 +139,10 @@ datapilot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
133
139
|
datapilot/utils/utils.py,sha256=MY8q6ZBJ0hkrTuH7gWMxAlEAQGrajXFMabEhtGtT7sc,11524
|
134
140
|
datapilot/utils/formatting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
135
141
|
datapilot/utils/formatting/utils.py,sha256=rAVmIYuldvw9VvCSwG2kMTEgiT7cEconp_F1sAWVyCo,1377
|
136
|
-
altimate_datapilot_cli-0.0.
|
137
|
-
altimate_datapilot_cli-0.0.
|
138
|
-
altimate_datapilot_cli-0.0.
|
139
|
-
altimate_datapilot_cli-0.0.
|
140
|
-
altimate_datapilot_cli-0.0.
|
141
|
-
altimate_datapilot_cli-0.0.
|
142
|
-
altimate_datapilot_cli-0.0.
|
142
|
+
altimate_datapilot_cli-0.0.14.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
|
143
|
+
altimate_datapilot_cli-0.0.14.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
|
144
|
+
altimate_datapilot_cli-0.0.14.dist-info/METADATA,sha256=s4DZVAhnfFUP7rj4HVF7H7Ote-5mozGCG8k6UySl5K4,2382
|
145
|
+
altimate_datapilot_cli-0.0.14.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
146
|
+
altimate_datapilot_cli-0.0.14.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
|
147
|
+
altimate_datapilot_cli-0.0.14.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
|
148
|
+
altimate_datapilot_cli-0.0.14.dist-info/RECORD,,
|
datapilot/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.0.
|
1
|
+
__version__ = "0.0.14"
|
@@ -44,7 +44,7 @@ class APIClient:
|
|
44
44
|
return response.json()
|
45
45
|
|
46
46
|
except HTTPError as http_err:
|
47
|
-
self.logger.
|
47
|
+
self.logger.debug(f"HTTP Error: {http_err.response.json()} - Status code: {response.status_code}")
|
48
48
|
except ConnectionError as conn_err:
|
49
49
|
self.logger.error(f"Connection error occurred: {conn_err}")
|
50
50
|
except Timeout as timeout_err:
|
@@ -84,6 +84,23 @@ class APIClient:
|
|
84
84
|
endpoint = "/dbt/v3/validate-credentials"
|
85
85
|
return self.get(endpoint)
|
86
86
|
|
87
|
+
def validate_upload_to_integration(self):
|
88
|
+
endpoint = "/dbt/v1/validate-permissions"
|
89
|
+
return self.get(endpoint)
|
90
|
+
|
87
91
|
def start_dbt_ingestion(self, params=None):
|
88
92
|
endpoint = "/dbt/v1/start_dbt_ingestion"
|
89
93
|
return self.post(endpoint, data=params)
|
94
|
+
|
95
|
+
def get_project_governance_llm_checks(self, params=None):
|
96
|
+
endpoint = "/project_governance/checks"
|
97
|
+
return self.get(endpoint, params=params)
|
98
|
+
|
99
|
+
def run_project_governance_llm_checks(self, manifest, catalog, check_names):
|
100
|
+
endpoint = "/project_governance/check/run"
|
101
|
+
data = {
|
102
|
+
"manifest": manifest,
|
103
|
+
"catalog": catalog,
|
104
|
+
"check_names": check_names,
|
105
|
+
}
|
106
|
+
return self.post(endpoint, data=data)
|
@@ -46,6 +46,15 @@ def validate_credentials(
|
|
46
46
|
return api_client.validate_credentials()
|
47
47
|
|
48
48
|
|
49
|
+
def validate_permissions(
|
50
|
+
token,
|
51
|
+
backend_url,
|
52
|
+
tenant,
|
53
|
+
) -> Response:
|
54
|
+
api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant)
|
55
|
+
return api_client.validate_upload_to_integration()
|
56
|
+
|
57
|
+
|
49
58
|
def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
|
50
59
|
api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
|
51
60
|
|
@@ -94,3 +103,24 @@ def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_int
|
|
94
103
|
"ok": False,
|
95
104
|
"message": "Error starting dbt ingestion worker. ",
|
96
105
|
}
|
106
|
+
|
107
|
+
|
108
|
+
def get_project_governance_llm_checks(
|
109
|
+
api_token,
|
110
|
+
tenant,
|
111
|
+
backend_url,
|
112
|
+
):
|
113
|
+
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
|
114
|
+
return api_client.get_project_governance_llm_checks()
|
115
|
+
|
116
|
+
|
117
|
+
def run_project_governance_llm_checks(
|
118
|
+
api_token,
|
119
|
+
tenant,
|
120
|
+
backend_url,
|
121
|
+
manifest,
|
122
|
+
catalog,
|
123
|
+
check_names,
|
124
|
+
):
|
125
|
+
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
|
126
|
+
return api_client.run_project_governance_llm_checks(manifest, catalog, check_names)
|
@@ -1,18 +1,11 @@
|
|
1
1
|
from abc import abstractmethod
|
2
|
-
from typing import Optional
|
3
2
|
|
4
|
-
from datapilot.core.insights.base
|
5
|
-
from datapilot.schemas.sql import Dialect
|
3
|
+
from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
|
6
4
|
|
7
5
|
|
8
|
-
class SqlInsight(
|
6
|
+
class SqlInsight(ChecksInsight):
|
9
7
|
NAME = "SqlInsight"
|
10
8
|
|
11
|
-
def __init__(self, sql: str, dialect: Optional[Dialect], *args, **kwargs):
|
12
|
-
self.sql = sql
|
13
|
-
self.dialect = dialect
|
14
|
-
super().__init__(*args, **kwargs)
|
15
|
-
|
16
9
|
@abstractmethod
|
17
10
|
def generate(self, *args, **kwargs) -> dict:
|
18
11
|
pass
|
@@ -6,6 +6,7 @@ from datapilot.clients.altimate.utils import check_token_and_instance
|
|
6
6
|
from datapilot.clients.altimate.utils import onboard_file
|
7
7
|
from datapilot.clients.altimate.utils import start_dbt_ingestion
|
8
8
|
from datapilot.clients.altimate.utils import validate_credentials
|
9
|
+
from datapilot.clients.altimate.utils import validate_permissions
|
9
10
|
from datapilot.config.config import load_config
|
10
11
|
from datapilot.core.platforms.dbt.constants import MODEL
|
11
12
|
from datapilot.core.platforms.dbt.constants import PROJECT
|
@@ -27,6 +28,8 @@ def dbt():
|
|
27
28
|
|
28
29
|
|
29
30
|
@dbt.command("project-health")
|
31
|
+
@click.option("--token", required=False, help="Your API token for authentication.")
|
32
|
+
@click.option("--instance-name", required=False, help="Your tenant ID.")
|
30
33
|
@click.option(
|
31
34
|
"--manifest-path",
|
32
35
|
required=True,
|
@@ -48,7 +51,10 @@ def dbt():
|
|
48
51
|
default=None,
|
49
52
|
help="Selective model testing. Specify one or more models to run tests on.",
|
50
53
|
)
|
51
|
-
|
54
|
+
@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
|
55
|
+
def project_health(
|
56
|
+
token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com"
|
57
|
+
):
|
52
58
|
"""
|
53
59
|
Validate the DBT project's configuration and structure.
|
54
60
|
:param manifest_path: Path to the DBT manifest file.
|
@@ -61,7 +67,16 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
|
|
61
67
|
selected_models = select.split(" ")
|
62
68
|
manifest = load_manifest(manifest_path)
|
63
69
|
catalog = load_catalog(catalog_path) if catalog_path else None
|
64
|
-
|
70
|
+
|
71
|
+
insight_generator = DBTInsightGenerator(
|
72
|
+
manifest=manifest,
|
73
|
+
catalog=catalog,
|
74
|
+
config=config,
|
75
|
+
selected_models=selected_models,
|
76
|
+
token=token,
|
77
|
+
instance_name=instance_name,
|
78
|
+
backend_url=backend_url,
|
79
|
+
)
|
65
80
|
reports = insight_generator.run()
|
66
81
|
|
67
82
|
package_insights = reports[PROJECT]
|
@@ -111,6 +126,10 @@ def onboard(
|
|
111
126
|
click.echo("Error: Invalid credentials.")
|
112
127
|
return
|
113
128
|
|
129
|
+
if not validate_permissions(token, backend_url, instance_name):
|
130
|
+
click.echo("Error: You don't have permission to perform this action.")
|
131
|
+
return
|
132
|
+
|
114
133
|
# This will throw error if manifest file is incorrect
|
115
134
|
try:
|
116
135
|
load_manifest(manifest_path)
|
@@ -5,11 +5,16 @@ from typing import Dict
|
|
5
5
|
from typing import List
|
6
6
|
from typing import Optional
|
7
7
|
|
8
|
+
from datapilot.clients.altimate.utils import get_project_governance_llm_checks
|
9
|
+
from datapilot.clients.altimate.utils import run_project_governance_llm_checks
|
10
|
+
from datapilot.core.platforms.dbt.constants import LLM
|
8
11
|
from datapilot.core.platforms.dbt.constants import MODEL
|
9
12
|
from datapilot.core.platforms.dbt.constants import PROJECT
|
10
13
|
from datapilot.core.platforms.dbt.exceptions import AltimateCLIArgumentError
|
11
14
|
from datapilot.core.platforms.dbt.factory import DBTFactory
|
12
15
|
from datapilot.core.platforms.dbt.insights import INSIGHTS
|
16
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
|
17
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
|
13
18
|
from datapilot.core.platforms.dbt.schemas.manifest import Catalog
|
14
19
|
from datapilot.core.platforms.dbt.schemas.manifest import Manifest
|
15
20
|
from datapilot.core.platforms.dbt.utils import get_models
|
@@ -29,11 +34,19 @@ class DBTInsightGenerator:
|
|
29
34
|
target: str = "dev",
|
30
35
|
selected_models: Optional[str] = None,
|
31
36
|
selected_model_ids: Optional[List[str]] = None,
|
37
|
+
token: Optional[str] = None,
|
38
|
+
instance_name: Optional[str] = None,
|
39
|
+
backend_url: Optional[str] = None,
|
32
40
|
):
|
33
41
|
self.run_results_path = run_results_path
|
34
42
|
self.target = target
|
35
43
|
self.env = env
|
36
44
|
self.config = config or {}
|
45
|
+
self.token = token
|
46
|
+
self.instance_name = instance_name
|
47
|
+
self.backend_url = backend_url
|
48
|
+
self.manifest = manifest
|
49
|
+
self.catalog = catalog
|
37
50
|
|
38
51
|
self.manifest_wrapper = DBTFactory.get_manifest_wrapper(manifest)
|
39
52
|
self.manifest_present = True
|
@@ -51,6 +64,7 @@ class DBTInsightGenerator:
|
|
51
64
|
self.macros = self.manifest_wrapper.get_macros()
|
52
65
|
self.sources = self.manifest_wrapper.get_sources()
|
53
66
|
self.exposures = self.manifest_wrapper.get_exposures()
|
67
|
+
self.adapter_type = self.manifest_wrapper.get_adapter_type()
|
54
68
|
self.seeds = self.manifest_wrapper.get_seeds()
|
55
69
|
self.children_map = self.manifest_wrapper.parent_to_child_map(self.nodes)
|
56
70
|
self.tests = self.manifest_wrapper.get_tests()
|
@@ -85,6 +99,22 @@ class DBTInsightGenerator:
|
|
85
99
|
return True
|
86
100
|
return False
|
87
101
|
|
102
|
+
def run_llm_checks(self):
|
103
|
+
llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url)
|
104
|
+
check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])]
|
105
|
+
if len(check_names) == 0:
|
106
|
+
return {"results": []}
|
107
|
+
|
108
|
+
llm_check_results = run_project_governance_llm_checks(
|
109
|
+
self.token,
|
110
|
+
self.instance_name,
|
111
|
+
self.backend_url,
|
112
|
+
self.manifest.json() if self.manifest else "",
|
113
|
+
self.catalog.json() if self.catalog else "",
|
114
|
+
check_names,
|
115
|
+
)
|
116
|
+
return llm_check_results
|
117
|
+
|
88
118
|
def run(self):
|
89
119
|
reports = {
|
90
120
|
MODEL: {},
|
@@ -112,6 +142,7 @@ class DBTInsightGenerator:
|
|
112
142
|
children_map=self.children_map,
|
113
143
|
tests=self.tests,
|
114
144
|
project_name=self.project_name,
|
145
|
+
adapter_type=self.adapter_type,
|
115
146
|
config=self.config,
|
116
147
|
selected_models=self.selected_models,
|
117
148
|
excluded_models=self.excluded_models,
|
@@ -154,4 +185,42 @@ class DBTInsightGenerator:
|
|
154
185
|
else:
|
155
186
|
self.logger.info(color_text(f"Skipping insight {insight_class.NAME} as {message}", YELLOW))
|
156
187
|
|
188
|
+
if self.token and self.instance_name and self.backend_url:
|
189
|
+
llm_check_results = self.run_llm_checks()
|
190
|
+
llm_reports = llm_check_results.get("results", [])
|
191
|
+
llm_insights = {}
|
192
|
+
for report in llm_reports:
|
193
|
+
for answer in report["answer"]:
|
194
|
+
location = answer["unique_id"]
|
195
|
+
if location not in llm_insights:
|
196
|
+
llm_insights[location] = []
|
197
|
+
metadata = answer.get("metadata", {})
|
198
|
+
metadata["source"] = LLM
|
199
|
+
metadata["teammate_check_id"] = report["id"]
|
200
|
+
metadata["category"] = report["type"]
|
201
|
+
llm_insights[location].append(
|
202
|
+
DBTModelInsightResponse(
|
203
|
+
insight=DBTInsightResult(
|
204
|
+
type="Custom",
|
205
|
+
name=report["name"],
|
206
|
+
message=answer["message"],
|
207
|
+
reason_to_flag=answer["reason_to_flag"],
|
208
|
+
recommendation=answer["recommendation"],
|
209
|
+
metadata=metadata,
|
210
|
+
),
|
211
|
+
severity=answer["severity"],
|
212
|
+
path=answer["path"] if answer.get("path") else "",
|
213
|
+
original_file_path=answer["original_file_path"] if answer.get("original_file_path") else "",
|
214
|
+
package_name=answer["package_name"] if answer.get("package_name") else "",
|
215
|
+
unique_id=answer["unique_id"],
|
216
|
+
)
|
217
|
+
)
|
218
|
+
|
219
|
+
if llm_insights:
|
220
|
+
for key, value in llm_insights.items():
|
221
|
+
if key in reports[MODEL]:
|
222
|
+
reports[MODEL][key].extend(value)
|
223
|
+
else:
|
224
|
+
reports[MODEL][key] = value
|
225
|
+
|
157
226
|
return reports
|
@@ -1,18 +1,22 @@
|
|
1
1
|
from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogV1
|
2
2
|
from dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10
|
3
3
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
|
4
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
|
4
5
|
|
5
6
|
from datapilot.core.platforms.dbt.schemas.manifest import Catalog
|
6
7
|
from datapilot.core.platforms.dbt.schemas.manifest import Manifest
|
7
8
|
from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import CatalogV1Wrapper
|
8
9
|
from datapilot.core.platforms.dbt.wrappers.manifest.v10.wrapper import ManifestV10Wrapper
|
9
10
|
from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import ManifestV11Wrapper
|
11
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.wrapper import ManifestV12Wrapper
|
10
12
|
from datapilot.exceptions.exceptions import AltimateNotSupportedError
|
11
13
|
|
12
14
|
|
13
15
|
class DBTFactory:
|
14
16
|
@classmethod
|
15
17
|
def get_manifest_wrapper(cls, manifest: Manifest):
|
18
|
+
if isinstance(manifest, ManifestV12):
|
19
|
+
return ManifestV12Wrapper(manifest)
|
16
20
|
if isinstance(manifest, ManifestV11):
|
17
21
|
return ManifestV11Wrapper(manifest)
|
18
22
|
if isinstance(manifest, ManifestV10):
|
@@ -51,6 +51,7 @@ from datapilot.core.platforms.dbt.insights.modelling.staging_model_dependent_on_
|
|
51
51
|
from datapilot.core.platforms.dbt.insights.modelling.unused_sources import DBTUnusedSources
|
52
52
|
from datapilot.core.platforms.dbt.insights.performance.chain_view_linking import DBTChainViewLinking
|
53
53
|
from datapilot.core.platforms.dbt.insights.performance.exposure_parent_materializations import DBTExposureParentMaterialization
|
54
|
+
from datapilot.core.platforms.dbt.insights.sql.sql_check import SqlCheck
|
54
55
|
from datapilot.core.platforms.dbt.insights.structure.model_directories_structure import DBTModelDirectoryStructure
|
55
56
|
from datapilot.core.platforms.dbt.insights.structure.model_naming_conventions import DBTModelNamingConvention
|
56
57
|
from datapilot.core.platforms.dbt.insights.structure.source_directories_structure import DBTSourceDirectoryStructure
|
@@ -112,4 +113,5 @@ INSIGHTS = [
|
|
112
113
|
CheckSourceHasTests,
|
113
114
|
CheckSourceTableHasDescription,
|
114
115
|
CheckSourceTags,
|
116
|
+
SqlCheck,
|
115
117
|
]
|
@@ -2,6 +2,7 @@ from abc import abstractmethod
|
|
2
2
|
from typing import ClassVar
|
3
3
|
from typing import Dict
|
4
4
|
from typing import List
|
5
|
+
from typing import Optional
|
5
6
|
from typing import Union
|
6
7
|
|
7
8
|
from datapilot.config.utils import get_insight_config
|
@@ -33,6 +34,7 @@ class DBTInsight(Insight):
|
|
33
34
|
macros: Dict[str, AltimateManifestMacroNode],
|
34
35
|
children_map: Dict[str, List[str]],
|
35
36
|
project_name: str,
|
37
|
+
adapter_type: Optional[str],
|
36
38
|
selected_models: Union[List[str], None] = None,
|
37
39
|
excluded_models: Union[List[str], None] = None,
|
38
40
|
*args,
|
@@ -47,6 +49,7 @@ class DBTInsight(Insight):
|
|
47
49
|
self.seeds = seeds
|
48
50
|
self.children_map = children_map
|
49
51
|
self.project_name = project_name
|
52
|
+
self.adapter_type = adapter_type
|
50
53
|
self.selected_models = selected_models
|
51
54
|
self.excluded_models = excluded_models
|
52
55
|
super().__init__(*args, **kwargs)
|
File without changes
|
@@ -0,0 +1,23 @@
|
|
1
|
+
from abc import abstractmethod
|
2
|
+
from typing import Tuple
|
3
|
+
|
4
|
+
from datapilot.core.platforms.dbt.insights.base import DBTInsight
|
5
|
+
|
6
|
+
|
7
|
+
class SqlInsight(DBTInsight):
|
8
|
+
TYPE = "governance"
|
9
|
+
|
10
|
+
@abstractmethod
|
11
|
+
def generate(self, *args, **kwargs) -> dict:
|
12
|
+
pass
|
13
|
+
|
14
|
+
@classmethod
|
15
|
+
def has_all_required_data(cls, has_manifest: bool, **kwargs) -> Tuple[bool, str]:
|
16
|
+
"""
|
17
|
+
Check if all required data is available for the insight to run.
|
18
|
+
:param has_manifest: A boolean indicating if manifest is available.
|
19
|
+
:return: A boolean indicating if all required data is available.
|
20
|
+
"""
|
21
|
+
if not has_manifest:
|
22
|
+
return False, "manifest is required for insight to run."
|
23
|
+
return True, ""
|
@@ -0,0 +1,101 @@
|
|
1
|
+
import inspect
|
2
|
+
from typing import List
|
3
|
+
|
4
|
+
from sqlglot import parse_one
|
5
|
+
from sqlglot.optimizer.eliminate_ctes import eliminate_ctes
|
6
|
+
from sqlglot.optimizer.eliminate_joins import eliminate_joins
|
7
|
+
from sqlglot.optimizer.eliminate_subqueries import eliminate_subqueries
|
8
|
+
from sqlglot.optimizer.normalize import normalize
|
9
|
+
from sqlglot.optimizer.pushdown_projections import pushdown_projections
|
10
|
+
from sqlglot.optimizer.qualify import qualify
|
11
|
+
from sqlglot.optimizer.unnest_subqueries import unnest_subqueries
|
12
|
+
|
13
|
+
from datapilot.core.insights.sql.base.insight import SqlInsight
|
14
|
+
from datapilot.core.insights.utils import get_severity
|
15
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
|
16
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
|
17
|
+
|
18
|
+
RULES = (
|
19
|
+
pushdown_projections,
|
20
|
+
normalize,
|
21
|
+
unnest_subqueries,
|
22
|
+
eliminate_subqueries,
|
23
|
+
eliminate_joins,
|
24
|
+
eliminate_ctes,
|
25
|
+
)
|
26
|
+
|
27
|
+
|
28
|
+
class SqlCheck(SqlInsight):
|
29
|
+
"""
|
30
|
+
This class identifies DBT models with SQL optimization issues.
|
31
|
+
"""
|
32
|
+
|
33
|
+
NAME = "sql optimization issues"
|
34
|
+
ALIAS = "check_sql_optimization"
|
35
|
+
DESCRIPTION = "Checks if the model has SQL optimization issues. "
|
36
|
+
REASON_TO_FLAG = "The query can be optimized."
|
37
|
+
FAILURE_MESSAGE = "The query for model `{model_unique_id}` has optimization opportunities:\n{rule_name}. "
|
38
|
+
RECOMMENDATION = "Please adapt the query of the model `{model_unique_id}` as in following example:\n{optimized_sql}"
|
39
|
+
|
40
|
+
def _build_failure_result(self, model_unique_id: str, rule_name: str, optimized_sql: str) -> DBTInsightResult:
|
41
|
+
"""
|
42
|
+
Constructs a failure result for a given model with sql optimization issues.
|
43
|
+
:param model_unique_id: The unique id of the dbt model.
|
44
|
+
:param rule_name: The rule that generated this failure result.
|
45
|
+
:param optimized_sql: The optimized sql.
|
46
|
+
:return: An instance of DBTInsightResult containing failure details.
|
47
|
+
"""
|
48
|
+
failure_message = self.FAILURE_MESSAGE.format(model_unique_id=model_unique_id, rule_name=rule_name)
|
49
|
+
recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id, optimized_sql=optimized_sql)
|
50
|
+
return DBTInsightResult(
|
51
|
+
type=self.TYPE,
|
52
|
+
name=self.NAME,
|
53
|
+
message=failure_message,
|
54
|
+
recommendation=recommendation,
|
55
|
+
reason_to_flag=self.REASON_TO_FLAG,
|
56
|
+
metadata={"model_unique_id": model_unique_id, "rule_name": rule_name},
|
57
|
+
)
|
58
|
+
|
59
|
+
def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
|
60
|
+
"""
|
61
|
+
Generates insights for each DBT model in the project, focusing on sql optimization issues.
|
62
|
+
|
63
|
+
:return: A list of DBTModelInsightResponse objects with insights for each model.
|
64
|
+
"""
|
65
|
+
self.logger.debug("Generating sql insights for DBT models")
|
66
|
+
insights = []
|
67
|
+
|
68
|
+
possible_kwargs = {
|
69
|
+
"db": None,
|
70
|
+
"catalog": None,
|
71
|
+
"dialect": self.adapter_type,
|
72
|
+
"isolate_tables": True, # needed for other optimizations to perform well
|
73
|
+
"quote_identifiers": False,
|
74
|
+
**kwargs,
|
75
|
+
}
|
76
|
+
for node_id, node in self.nodes.items():
|
77
|
+
try:
|
78
|
+
compiled_query = node.compiled_code
|
79
|
+
if compiled_query:
|
80
|
+
parsed_query = parse_one(compiled_query, dialect=self.adapter_type)
|
81
|
+
qualified = qualify(parsed_query, **possible_kwargs)
|
82
|
+
changed = qualified.copy()
|
83
|
+
for rule in RULES:
|
84
|
+
original = changed.copy()
|
85
|
+
rule_params = inspect.getfullargspec(rule).args
|
86
|
+
rule_kwargs = {param: possible_kwargs[param] for param in rule_params if param in possible_kwargs}
|
87
|
+
changed = rule(changed, **rule_kwargs)
|
88
|
+
if changed.sql() != original.sql():
|
89
|
+
insights.append(
|
90
|
+
DBTModelInsightResponse(
|
91
|
+
unique_id=node_id,
|
92
|
+
package_name=node.package_name,
|
93
|
+
path=node.original_file_path,
|
94
|
+
original_file_path=node.original_file_path,
|
95
|
+
insight=self._build_failure_result(node_id, rule.__name__, changed.sql()),
|
96
|
+
severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
|
97
|
+
)
|
98
|
+
)
|
99
|
+
except Exception as e:
|
100
|
+
self.logger.error(e)
|
101
|
+
return insights
|
@@ -1,4 +1,5 @@
|
|
1
1
|
from typing import Dict
|
2
|
+
from typing import Optional
|
2
3
|
from typing import Set
|
3
4
|
|
4
5
|
from dbt_artifacts_parser.parsers.manifest.manifest_v10 import GenericTestNode
|
@@ -67,6 +68,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
|
|
67
68
|
depends_on_macros = node.depends_on.macros if node.depends_on else None
|
68
69
|
compiled_path = node.compiled_path
|
69
70
|
compiled = node.compiled
|
71
|
+
compiled_code = node.compiled_code
|
70
72
|
raw_code = node.raw_code
|
71
73
|
language = node.language
|
72
74
|
contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
|
@@ -381,6 +383,9 @@ class ManifestV10Wrapper(BaseManifestWrapper):
|
|
381
383
|
seeds[seed.unique_id] = self._get_seed(seed)
|
382
384
|
return seeds
|
383
385
|
|
386
|
+
def get_adapter_type(self) -> Optional[str]:
|
387
|
+
return self.manifest.metadata.adapter_type
|
388
|
+
|
384
389
|
def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
|
385
390
|
"""
|
386
391
|
Current manifest contains information about parents
|
@@ -1,4 +1,5 @@
|
|
1
1
|
from typing import Dict
|
2
|
+
from typing import Optional
|
2
3
|
from typing import Set
|
3
4
|
|
4
5
|
from dbt_artifacts_parser.parsers.manifest.manifest_v11 import GenericTestNode
|
@@ -67,6 +68,7 @@ class ManifestV11Wrapper(BaseManifestWrapper):
|
|
67
68
|
depends_on_macros = node.depends_on.macros if node.depends_on else None
|
68
69
|
compiled_path = node.compiled_path
|
69
70
|
compiled = node.compiled
|
71
|
+
compiled_code = node.compiled_code
|
70
72
|
raw_code = node.raw_code
|
71
73
|
language = node.language
|
72
74
|
contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
|
@@ -381,6 +383,9 @@ class ManifestV11Wrapper(BaseManifestWrapper):
|
|
381
383
|
seeds[seed.unique_id] = self._get_seed(seed)
|
382
384
|
return seeds
|
383
385
|
|
386
|
+
def get_adapter_type(self) -> Optional[str]:
|
387
|
+
return self.manifest.metadata.adapter_type
|
388
|
+
|
384
389
|
def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
|
385
390
|
"""
|
386
391
|
Current manifest contains information about parents
|
File without changes
|
@@ -0,0 +1,36 @@
|
|
1
|
+
from typing import Dict
|
2
|
+
from typing import Type
|
3
|
+
from typing import Union
|
4
|
+
|
5
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Exposures
|
6
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Macros
|
7
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node
|
8
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node1
|
9
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2
|
10
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node3
|
11
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node4
|
12
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node5
|
13
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6
|
14
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node7
|
15
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Sources
|
16
|
+
|
17
|
+
from datapilot.core.platforms.dbt.constants import GENERIC
|
18
|
+
from datapilot.core.platforms.dbt.constants import SINGULAR
|
19
|
+
|
20
|
+
ManifestNode = Union[Node, Node1, Node2, Node3, Node4, Node5, Node6, Node7]
|
21
|
+
|
22
|
+
SourceNode = Sources
|
23
|
+
|
24
|
+
ExposureNode = Exposures
|
25
|
+
|
26
|
+
TestNode = Union[Node6, Node2]
|
27
|
+
|
28
|
+
MacroNode = Macros
|
29
|
+
|
30
|
+
TEST_TYPE_TO_NODE_MAP: Dict[str, Type] = {
|
31
|
+
GENERIC: [Node6],
|
32
|
+
SINGULAR: [Node2],
|
33
|
+
}
|
34
|
+
|
35
|
+
|
36
|
+
SeedNodeMap = Node
|
@@ -0,0 +1,414 @@
|
|
1
|
+
from typing import Dict
|
2
|
+
from typing import Optional
|
3
|
+
from typing import Set
|
4
|
+
|
5
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
|
6
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2
|
7
|
+
from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6
|
8
|
+
|
9
|
+
from datapilot.core.platforms.dbt.constants import GENERIC
|
10
|
+
from datapilot.core.platforms.dbt.constants import OTHER_TEST_NODE
|
11
|
+
from datapilot.core.platforms.dbt.constants import SEED
|
12
|
+
from datapilot.core.platforms.dbt.constants import SINGULAR
|
13
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateDBTContract
|
14
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateDependsOn
|
15
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateExposureType
|
16
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateExternalTable
|
17
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateFileHash
|
18
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateFreshnessThreshold
|
19
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateMacroArgument
|
20
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestColumnInfo
|
21
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestExposureNode
|
22
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestMacroNode
|
23
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestNode
|
24
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestSourceNode
|
25
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestTestNode
|
26
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateMaturityEnum
|
27
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateNodeConfig
|
28
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateOwner
|
29
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateQuoting
|
30
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateRefArgs
|
31
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
|
32
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedConfig
|
33
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedNode
|
34
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateSourceConfig
|
35
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestConfig
|
36
|
+
from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestMetadata
|
37
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import TEST_TYPE_TO_NODE_MAP
|
38
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import ExposureNode
|
39
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import MacroNode
|
40
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import ManifestNode
|
41
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import SeedNodeMap
|
42
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import SourceNode
|
43
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import TestNode
|
44
|
+
from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper
|
45
|
+
|
46
|
+
|
47
|
+
class ManifestV12Wrapper(BaseManifestWrapper):
|
48
|
+
def __init__(self, manifest: ManifestV12):
|
49
|
+
self.manifest = manifest
|
50
|
+
|
51
|
+
def _get_node(self, node: ManifestNode) -> AltimateManifestNode:
|
52
|
+
(
|
53
|
+
sources,
|
54
|
+
metrics,
|
55
|
+
compiled_path,
|
56
|
+
compiled,
|
57
|
+
compiled_code,
|
58
|
+
depends_on_nodes,
|
59
|
+
depends_on_macros,
|
60
|
+
raw_code,
|
61
|
+
language,
|
62
|
+
contract,
|
63
|
+
) = ([], [], None, None, None, None, None, "", "", None)
|
64
|
+
if node.resource_type != SEED:
|
65
|
+
sources = node.sources
|
66
|
+
metrics = node.metrics
|
67
|
+
depends_on_nodes = node.depends_on.nodes if node.depends_on else None
|
68
|
+
depends_on_macros = node.depends_on.macros if node.depends_on else None
|
69
|
+
compiled_path = node.compiled_path
|
70
|
+
compiled = node.compiled
|
71
|
+
compiled_code = node.compiled_code
|
72
|
+
raw_code = node.raw_code
|
73
|
+
language = node.language
|
74
|
+
contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
|
75
|
+
|
76
|
+
return AltimateManifestNode(
|
77
|
+
database=node.database,
|
78
|
+
schema_name=node.schema_,
|
79
|
+
name=node.name,
|
80
|
+
resource_type=AltimateResourceType(node.resource_type),
|
81
|
+
package_name=node.package_name,
|
82
|
+
path=node.path,
|
83
|
+
description=node.description,
|
84
|
+
original_file_path=node.original_file_path,
|
85
|
+
unique_id=node.unique_id,
|
86
|
+
fqn=node.fqn,
|
87
|
+
alias=node.alias,
|
88
|
+
raw_code=raw_code,
|
89
|
+
language=language,
|
90
|
+
config=AltimateNodeConfig(**node.config.__dict__) if node.config else None,
|
91
|
+
checksum=AltimateFileHash(
|
92
|
+
name=node.checksum.name if node.checksum else None,
|
93
|
+
checksum=node.checksum.checksum if node.checksum else None,
|
94
|
+
),
|
95
|
+
columns={
|
96
|
+
name: AltimateManifestColumnInfo(
|
97
|
+
name=column.name,
|
98
|
+
description=column.description,
|
99
|
+
meta=column.meta,
|
100
|
+
data_type=column.data_type,
|
101
|
+
quote=column.quote,
|
102
|
+
tags=column.tags,
|
103
|
+
)
|
104
|
+
for name, column in node.columns.items()
|
105
|
+
},
|
106
|
+
relation_name=node.relation_name,
|
107
|
+
sources=sources,
|
108
|
+
metrics=metrics,
|
109
|
+
depends_on=AltimateDependsOn(
|
110
|
+
nodes=depends_on_nodes,
|
111
|
+
macros=depends_on_macros,
|
112
|
+
),
|
113
|
+
compiled_path=compiled_path,
|
114
|
+
compiled=compiled,
|
115
|
+
compiled_code=compiled_code,
|
116
|
+
contract=contract,
|
117
|
+
meta=node.meta,
|
118
|
+
patch_path=node.patch_path,
|
119
|
+
)
|
120
|
+
|
121
|
+
def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
|
122
|
+
return AltimateManifestSourceNode(
|
123
|
+
database=source.database,
|
124
|
+
resource_type=AltimateResourceType(source.resource_type),
|
125
|
+
schema_name=source.schema_,
|
126
|
+
name=source.name,
|
127
|
+
package_name=source.package_name,
|
128
|
+
path=source.path,
|
129
|
+
original_file_path=source.original_file_path,
|
130
|
+
unique_id=source.unique_id,
|
131
|
+
fqn=source.fqn,
|
132
|
+
source_name=source.source_name,
|
133
|
+
source_description=source.source_description,
|
134
|
+
loader=source.loader,
|
135
|
+
identifier=source.identifier,
|
136
|
+
quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None,
|
137
|
+
loaded_at_field=source.loaded_at_field,
|
138
|
+
freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None,
|
139
|
+
external=AltimateExternalTable(**source.external.dict()) if source.external else None,
|
140
|
+
description=source.description,
|
141
|
+
columns={
|
142
|
+
name: AltimateManifestColumnInfo(
|
143
|
+
name=column.name,
|
144
|
+
description=column.description,
|
145
|
+
meta=column.meta,
|
146
|
+
data_type=column.data_type,
|
147
|
+
quote=column.quote,
|
148
|
+
tags=column.tags,
|
149
|
+
)
|
150
|
+
for name, column in source.columns.items()
|
151
|
+
},
|
152
|
+
meta=source.meta,
|
153
|
+
relation_name=source.relation_name,
|
154
|
+
source_meta=source.source_meta,
|
155
|
+
tags=source.tags,
|
156
|
+
config=AltimateSourceConfig(**source.config.dict()) if source.config else None,
|
157
|
+
patch_path=source.patch_path,
|
158
|
+
unrendered_config=source.unrendered_config,
|
159
|
+
created_at=source.created_at,
|
160
|
+
)
|
161
|
+
|
162
|
+
def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode:
|
163
|
+
return AltimateManifestMacroNode(
|
164
|
+
name=macro.name,
|
165
|
+
resource_type=AltimateResourceType(macro.resource_type),
|
166
|
+
package_name=macro.package_name,
|
167
|
+
path=macro.path,
|
168
|
+
original_file_path=macro.original_file_path,
|
169
|
+
unique_id=macro.unique_id,
|
170
|
+
macro_sql=macro.macro_sql,
|
171
|
+
depends_on=(
|
172
|
+
AltimateDependsOn(
|
173
|
+
macros=macro.depends_on.macros,
|
174
|
+
)
|
175
|
+
if macro.depends_on
|
176
|
+
else None
|
177
|
+
),
|
178
|
+
description=macro.description,
|
179
|
+
meta=macro.meta,
|
180
|
+
docs=macro.docs,
|
181
|
+
patch_path=macro.patch_path,
|
182
|
+
arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None,
|
183
|
+
created_at=macro.created_at,
|
184
|
+
supported_languages=macro.supported_languages,
|
185
|
+
)
|
186
|
+
|
187
|
+
def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode:
|
188
|
+
return AltimateManifestExposureNode(
|
189
|
+
name=exposure.name,
|
190
|
+
resource_type=AltimateResourceType(exposure.resource_type),
|
191
|
+
package_name=exposure.package_name,
|
192
|
+
path=exposure.path,
|
193
|
+
original_file_path=exposure.original_file_path,
|
194
|
+
unique_id=exposure.unique_id,
|
195
|
+
fqn=exposure.fqn,
|
196
|
+
type=AltimateExposureType(exposure.type.value) if exposure.type else None,
|
197
|
+
owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None,
|
198
|
+
description=exposure.description,
|
199
|
+
label=exposure.label,
|
200
|
+
maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None,
|
201
|
+
meta=exposure.meta,
|
202
|
+
tags=exposure.tags,
|
203
|
+
config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None,
|
204
|
+
unrendered_config=exposure.unrendered_config,
|
205
|
+
url=exposure.url,
|
206
|
+
depends_on=(
|
207
|
+
AltimateDependsOn(
|
208
|
+
nodes=exposure.depends_on.nodes,
|
209
|
+
macros=exposure.depends_on.macros,
|
210
|
+
)
|
211
|
+
if exposure.depends_on
|
212
|
+
else None
|
213
|
+
),
|
214
|
+
refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None,
|
215
|
+
sources=exposure.sources,
|
216
|
+
metrics=exposure.metrics,
|
217
|
+
created_at=exposure.created_at,
|
218
|
+
)
|
219
|
+
|
220
|
+
def _get_tests(self, test: TestNode) -> AltimateManifestTestNode:
|
221
|
+
test_metadata = None
|
222
|
+
if isinstance(test, Node6):
|
223
|
+
test_type = GENERIC
|
224
|
+
test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None
|
225
|
+
elif isinstance(test, Node2):
|
226
|
+
test_type = SINGULAR
|
227
|
+
else:
|
228
|
+
test_type = OTHER_TEST_NODE
|
229
|
+
return AltimateManifestTestNode(
|
230
|
+
test_metadata=test_metadata,
|
231
|
+
test_type=test_type,
|
232
|
+
name=test.name,
|
233
|
+
resource_type=AltimateResourceType(test.resource_type),
|
234
|
+
package_name=test.package_name,
|
235
|
+
path=test.path,
|
236
|
+
original_file_path=test.original_file_path,
|
237
|
+
unique_id=test.unique_id,
|
238
|
+
fqn=test.fqn,
|
239
|
+
alias=test.alias,
|
240
|
+
checksum=(
|
241
|
+
AltimateFileHash(
|
242
|
+
name=test.checksum.name,
|
243
|
+
checksum=test.checksum.checksum,
|
244
|
+
)
|
245
|
+
if test.checksum
|
246
|
+
else None
|
247
|
+
),
|
248
|
+
config=AltimateTestConfig(**test.config.dict()) if test.config else None,
|
249
|
+
description=test.description,
|
250
|
+
tags=test.tags,
|
251
|
+
columns=(
|
252
|
+
{
|
253
|
+
name: AltimateManifestColumnInfo(
|
254
|
+
name=column.name,
|
255
|
+
description=column.description,
|
256
|
+
meta=column.meta,
|
257
|
+
data_type=column.data_type,
|
258
|
+
quote=column.quote,
|
259
|
+
tags=column.tags,
|
260
|
+
)
|
261
|
+
for name, column in test.columns.items()
|
262
|
+
}
|
263
|
+
if test.columns
|
264
|
+
else None
|
265
|
+
),
|
266
|
+
meta=test.meta,
|
267
|
+
relation_name=test.relation_name,
|
268
|
+
group=test.group,
|
269
|
+
raw_code=test.raw_code,
|
270
|
+
language=test.language,
|
271
|
+
refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None,
|
272
|
+
sources=test.sources,
|
273
|
+
metrics=test.metrics,
|
274
|
+
depends_on=(
|
275
|
+
AltimateDependsOn(
|
276
|
+
nodes=test.depends_on.nodes,
|
277
|
+
macros=test.depends_on.macros,
|
278
|
+
)
|
279
|
+
if test.depends_on
|
280
|
+
else None
|
281
|
+
),
|
282
|
+
compiled_path=test.compiled_path,
|
283
|
+
compiled=test.compiled,
|
284
|
+
compiled_code=test.compiled_code,
|
285
|
+
)
|
286
|
+
|
287
|
+
def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode:
|
288
|
+
return AltimateSeedNode(
|
289
|
+
database=seed.database,
|
290
|
+
schema_name=seed.schema_,
|
291
|
+
name=seed.name,
|
292
|
+
resource_type=AltimateResourceType(seed.resource_type),
|
293
|
+
package_name=seed.package_name,
|
294
|
+
path=seed.path,
|
295
|
+
original_file_path=seed.original_file_path,
|
296
|
+
unique_id=seed.unique_id,
|
297
|
+
fqn=seed.fqn,
|
298
|
+
alias=seed.alias,
|
299
|
+
checksum=(
|
300
|
+
AltimateFileHash(
|
301
|
+
name=seed.checksum.name,
|
302
|
+
checksum=seed.checksum.checksum,
|
303
|
+
)
|
304
|
+
if seed.checksum
|
305
|
+
else None
|
306
|
+
),
|
307
|
+
config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None,
|
308
|
+
description=seed.description,
|
309
|
+
tags=seed.tags,
|
310
|
+
columns=(
|
311
|
+
{
|
312
|
+
name: AltimateManifestColumnInfo(
|
313
|
+
name=column.name,
|
314
|
+
description=column.description,
|
315
|
+
meta=column.meta,
|
316
|
+
data_type=column.data_type,
|
317
|
+
quote=column.quote,
|
318
|
+
tags=column.tags,
|
319
|
+
)
|
320
|
+
for name, column in seed.columns.items()
|
321
|
+
}
|
322
|
+
if seed.columns
|
323
|
+
else None
|
324
|
+
),
|
325
|
+
meta=seed.meta,
|
326
|
+
group=seed.group,
|
327
|
+
docs=seed.docs.dict() if seed.docs else None,
|
328
|
+
patch_path=seed.patch_path,
|
329
|
+
build_path=seed.build_path,
|
330
|
+
deferred=False,
|
331
|
+
unrendered_config=seed.unrendered_config,
|
332
|
+
created_at=seed.created_at,
|
333
|
+
config_call_dict=seed.config_call_dict,
|
334
|
+
)
|
335
|
+
|
336
|
+
def get_nodes(
|
337
|
+
self,
|
338
|
+
) -> Dict[str, AltimateManifestNode]:
|
339
|
+
nodes = {}
|
340
|
+
for node in self.manifest.nodes.values():
|
341
|
+
if (
|
342
|
+
node.resource_type
|
343
|
+
in [
|
344
|
+
AltimateResourceType.seed.value,
|
345
|
+
AltimateResourceType.test.value,
|
346
|
+
]
|
347
|
+
or node.package_name != self.get_package()
|
348
|
+
):
|
349
|
+
continue
|
350
|
+
nodes[node.unique_id] = self._get_node(node)
|
351
|
+
return nodes
|
352
|
+
|
353
|
+
def get_package(self) -> str:
|
354
|
+
return self.manifest.metadata.project_name
|
355
|
+
|
356
|
+
def get_sources(self) -> Dict[str, AltimateManifestSourceNode]:
|
357
|
+
sources = {}
|
358
|
+
for source in self.manifest.sources.values():
|
359
|
+
sources[source.unique_id] = self._get_source(source)
|
360
|
+
return sources
|
361
|
+
|
362
|
+
def get_macros(self) -> Dict[str, AltimateManifestMacroNode]:
|
363
|
+
macros = {}
|
364
|
+
for macro in self.manifest.macros.values():
|
365
|
+
if macro.resource_type == AltimateResourceType.macro.value and macro.package_name == self.get_package():
|
366
|
+
macros[macro.unique_id] = self._get_macro(macro)
|
367
|
+
return macros
|
368
|
+
|
369
|
+
def get_exposures(self) -> Dict[str, AltimateManifestExposureNode]:
|
370
|
+
exposures = {}
|
371
|
+
for exposure in self.manifest.exposures.values():
|
372
|
+
exposures[exposure.unique_id] = self._get_exposure(exposure)
|
373
|
+
return exposures
|
374
|
+
|
375
|
+
def get_tests(self, type=None) -> Dict[str, AltimateManifestTestNode]:
|
376
|
+
tests = {}
|
377
|
+
# Initialize types_union with TestNode
|
378
|
+
types = [Node2, Node6]
|
379
|
+
|
380
|
+
# Add other types to the union if provided
|
381
|
+
if type:
|
382
|
+
types = TEST_TYPE_TO_NODE_MAP.get(type)
|
383
|
+
|
384
|
+
for node in self.manifest.nodes.values():
|
385
|
+
# Check if the node is a test and of the correct type
|
386
|
+
if node.resource_type == AltimateResourceType.test.value:
|
387
|
+
if any(isinstance(node, t) for t in types):
|
388
|
+
tests[node.unique_id] = self._get_tests(node)
|
389
|
+
return tests
|
390
|
+
|
391
|
+
def get_seeds(self) -> Dict[str, AltimateSeedNode]:
|
392
|
+
seeds = {}
|
393
|
+
for seed in self.manifest.nodes.values():
|
394
|
+
if seed.resource_type == AltimateResourceType.seed.value:
|
395
|
+
seeds[seed.unique_id] = self._get_seed(seed)
|
396
|
+
return seeds
|
397
|
+
|
398
|
+
def get_adapter_type(self) -> Optional[str]:
|
399
|
+
return self.manifest.metadata.adapter_type
|
400
|
+
|
401
|
+
def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
|
402
|
+
"""
|
403
|
+
Current manifest contains information about parents
|
404
|
+
THis gives an information of node to childre
|
405
|
+
:param nodes: A dictionary of nodes in a manifest.
|
406
|
+
:return: A dictionary of all the children of a node.
|
407
|
+
"""
|
408
|
+
children_map = {}
|
409
|
+
for node_id, node in nodes.items():
|
410
|
+
if node_id not in children_map:
|
411
|
+
children_map[node_id] = set()
|
412
|
+
for parent in node.depends_on.nodes or []:
|
413
|
+
children_map.setdefault(parent, set()).add(node_id)
|
414
|
+
return children_map
|
@@ -1,6 +1,7 @@
|
|
1
1
|
from abc import ABC
|
2
2
|
from abc import abstractmethod
|
3
3
|
from typing import Dict
|
4
|
+
from typing import Optional
|
4
5
|
from typing import Set
|
5
6
|
|
6
7
|
from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestExposureNode
|
@@ -26,6 +27,10 @@ class BaseManifestWrapper(ABC):
|
|
26
27
|
def get_exposures(self) -> Dict[str, AltimateManifestExposureNode]:
|
27
28
|
pass
|
28
29
|
|
30
|
+
@abstractmethod
|
31
|
+
def get_adapter_type(self) -> Optional[str]:
|
32
|
+
pass
|
33
|
+
|
29
34
|
@abstractmethod
|
30
35
|
def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
|
31
36
|
pass
|
{altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/AUTHORS.rst
RENAMED
File without changes
|
File without changes
|
{altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/entry_points.txt
RENAMED
File without changes
|
{altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/top_level.txt
RENAMED
File without changes
|