altimate-datapilot-cli 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/METADATA +7 -6
  2. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/RECORD +26 -20
  3. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/WHEEL +1 -1
  4. datapilot/__init__.py +1 -1
  5. datapilot/clients/altimate/client.py +18 -1
  6. datapilot/clients/altimate/utils.py +30 -0
  7. datapilot/core/insights/sql/base/insight.py +2 -9
  8. datapilot/core/platforms/dbt/cli/cli.py +21 -2
  9. datapilot/core/platforms/dbt/constants.py +2 -0
  10. datapilot/core/platforms/dbt/executor.py +69 -0
  11. datapilot/core/platforms/dbt/factory.py +4 -0
  12. datapilot/core/platforms/dbt/insights/__init__.py +2 -0
  13. datapilot/core/platforms/dbt/insights/base.py +3 -0
  14. datapilot/core/platforms/dbt/insights/sql/__init__.py +0 -0
  15. datapilot/core/platforms/dbt/insights/sql/base.py +23 -0
  16. datapilot/core/platforms/dbt/insights/sql/sql_check.py +101 -0
  17. datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +5 -0
  18. datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +5 -0
  19. datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py +0 -0
  20. datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py +36 -0
  21. datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py +414 -0
  22. datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py +5 -0
  23. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/AUTHORS.rst +0 -0
  24. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/LICENSE +0 -0
  25. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/entry_points.txt +0 -0
  26. {altimate_datapilot_cli-0.0.12.dist-info → altimate_datapilot_cli-0.0.14.dist-info}/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: altimate-datapilot-cli
3
- Version: 0.0.12
3
+ Version: 0.0.14
4
4
  Summary: Assistant for Data Teams
5
5
  Home-page: https://github.com/AltimateAI/datapilot-cli
6
6
  Author: Altimate Inc
@@ -30,11 +30,12 @@ Classifier: Topic :: Utilities
30
30
  Requires-Python: >=3.8
31
31
  License-File: LICENSE
32
32
  License-File: AUTHORS.rst
33
- Requires-Dist: click (==8.1.7)
34
- Requires-Dist: dbt-artifacts-parser (==0.5.1)
35
- Requires-Dist: ruamel.yaml (==0.18.6)
36
- Requires-Dist: tabulate (==0.9.0)
37
- Requires-Dist: requests (==2.31.0)
33
+ Requires-Dist: click==8.1.7
34
+ Requires-Dist: dbt-artifacts-parser==0.6.0
35
+ Requires-Dist: ruamel.yaml==0.18.6
36
+ Requires-Dist: tabulate==0.9.0
37
+ Requires-Dist: requests==2.31.0
38
+ Requires-Dist: sqlglot==25.30.0
38
39
 
39
40
  ========
40
41
  Overview
@@ -1,11 +1,11 @@
1
- datapilot/__init__.py,sha256=1SFj-qMOaeO0fOKQaFmYhAKiRwwNda5uWKpYtIcken8,23
1
+ datapilot/__init__.py,sha256=71Tw8stu19MsLPyF1q_DF5mb_BefzOTPYggLlSLvCms,23
2
2
  datapilot/__main__.py,sha256=I9USmeNnK-cAHb6LZfydJC0LeNSE8enieeY55wpR6uw,380
3
3
  datapilot/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
4
  datapilot/cli/main.py,sha256=VSdqlkCiu8GSG9qQh8q0BzyocsQc4lKWxZAPEsjXF18,181
5
5
  datapilot/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
6
  datapilot/clients/altimate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
- datapilot/clients/altimate/client.py,sha256=DHPG2y7r1gFph80TVp7zozd_Hl05mzxtNFQ4JYEN1Jk,3260
8
- datapilot/clients/altimate/utils.py,sha256=IyS4iY5nE5KF9bivHKVzioUitQnmgVVIZjqPNFoeUwY,3547
7
+ datapilot/clients/altimate/client.py,sha256=lLIb9wu--5Rk3qpgnmqeyHlWiPmiTIE-nyMXdWodQ5I,3875
8
+ datapilot/clients/altimate/utils.py,sha256=B94GQgaiBbw4GrLrK8P7r89zzVeYIzr9-8tuJhoN2MY,4302
9
9
  datapilot/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
10
  datapilot/config/config.py,sha256=kyj53Qsb85V4iGQsX0vSwULOjscMOSFrJDJ3tnagJpo,403
11
11
  datapilot/config/utils.py,sha256=DIAVX-OZ5Lc0Ky_A7dvdbPcD1QSg2DRxZcuaIIZ2rhw,1146
@@ -18,23 +18,23 @@ datapilot/core/insights/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
18
18
  datapilot/core/insights/base/insight.py,sha256=AshjeLhV7cxwYANfSidDfHZssifTmvymkGHcWkeKipk,883
19
19
  datapilot/core/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
20
  datapilot/core/insights/sql/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
- datapilot/core/insights/sql/base/insight.py,sha256=fU7Pm7AkytlafCROEIFY1mDvLf31V_hkNlL5Gy45v4M,471
21
+ datapilot/core/insights/sql/base/insight.py,sha256=k8UUn0qrN-QG6NCunPl7Hd6L6kd1X1eUAeGEsyl8v0o,250
22
22
  datapilot/core/insights/sql/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
23
  datapilot/core/insights/sql/static/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
24
  datapilot/core/platforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
25
  datapilot/core/platforms/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
26
- datapilot/core/platforms/dbt/constants.py,sha256=N9Ovo9100iOAeRu8pPZLYg3_11O5QKgs3dW0VMU6jR8,502
26
+ datapilot/core/platforms/dbt/constants.py,sha256=kW4PEsYWosfDjBEZ9JUhWFtMxKMSNoIn0DNPlJEJuYc,515
27
27
  datapilot/core/platforms/dbt/exceptions.py,sha256=IC5BgcU90gjYYwPcfTlPNtn0_p8fYjavDRMpKZQ0OnY,110
28
- datapilot/core/platforms/dbt/executor.py,sha256=hvdh0qJjalWJN8ElJjIUvUiLrMIcvGvoQfrysMTqa-Y,6562
29
- datapilot/core/platforms/dbt/factory.py,sha256=fsk8XWXiSq22t5BH5jGQMASWIJ-eUNqOh5x551cbnLw,1331
28
+ datapilot/core/platforms/dbt/executor.py,sha256=4bRNg6yxU1rHu5MDGdq1vO-A0EKS38MNP1hAAkO5h6Y,9992
29
+ datapilot/core/platforms/dbt/factory.py,sha256=YIQtb-FQQAJsifJ3KiLjjk0WIKTHtEPTNu2MeKHdMG8,1590
30
30
  datapilot/core/platforms/dbt/formatting.py,sha256=bpfa7XmVghTq4WnGDGYC6DruwOwH8YmjFHghoo5cPD8,1638
31
31
  datapilot/core/platforms/dbt/utils.py,sha256=ozFHprR6LTLXQdrGyaRoyIBTua4P1NkP8T7LGgN-9c0,18577
32
32
  datapilot/core/platforms/dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
- datapilot/core/platforms/dbt/cli/cli.py,sha256=qzeJU8ALoo-VykKeqPT0O5pheMMH9noBh7S42099p7w,5563
33
+ datapilot/core/platforms/dbt/cli/cli.py,sha256=1phfriG4f5pS1UFwBcZ0uM49iovVpnjdama9lSYcKMU,6266
34
34
  datapilot/core/platforms/dbt/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
35
  datapilot/core/platforms/dbt/hooks/executor_hook.py,sha256=gSM50vAO7C-f1rdnHogWbqc87aCXPXysZepjp5L2qzw,2966
36
- datapilot/core/platforms/dbt/insights/__init__.py,sha256=vnNOqP6lJ-r1SqmxQbeTEphL-omj466OanqQY1WNrUA,7547
37
- datapilot/core/platforms/dbt/insights/base.py,sha256=XA4hAUf0wfTId0p7gEdBRfz1puNGh_6rRw_YV3rjMEI,5260
36
+ datapilot/core/platforms/dbt/insights/__init__.py,sha256=hk7BAzCTDkY8WNV6L0v-CPn9mrsDyJJusoQxNxGyzAY,7634
37
+ datapilot/core/platforms/dbt/insights/base.py,sha256=WBp3knDE2GJx-aM-gHhOWWSu1mdAJHzxLd243sLRdRw,5366
38
38
  datapilot/core/platforms/dbt/insights/schema.py,sha256=4nPxEGsgN5sCXQ1BrOesTRphkMKZj1szyk6ckoz49eg,2657
39
39
  datapilot/core/platforms/dbt/insights/utils.py,sha256=2btHwxzxEm7cjoLSzzbnZ3q3caI-58NJeWM0_TouJhI,267
40
40
  datapilot/core/platforms/dbt/insights/checks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -99,6 +99,9 @@ datapilot/core/platforms/dbt/insights/performance/__init__.py,sha256=47DEQpj8HBS
99
99
  datapilot/core/platforms/dbt/insights/performance/base.py,sha256=4EE7WmwlqMQSmBcpYeZrellvGU87F81cTL5wxBV5QSo,829
100
100
  datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py,sha256=gqL4qqNAybOTYhc9VGlrbQQJ1PkDtBHFKrcQi_XRpTM,3627
101
101
  datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py,sha256=bX7yqls9eMHShXeNdZ8J3IefM3Gp-4D9OGXAYfrKHEs,4689
102
+ datapilot/core/platforms/dbt/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
+ datapilot/core/platforms/dbt/insights/sql/base.py,sha256=dpmHtX3rg1_ewECcsdiVkndftK0EBOnvOfsLw2TUg8U,733
104
+ datapilot/core/platforms/dbt/insights/sql/sql_check.py,sha256=a84YLtjJFtF1NRBbWueSdAwp6NNi6ZU8_ScbwLfpnfk,4652
102
105
  datapilot/core/platforms/dbt/insights/structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
106
  datapilot/core/platforms/dbt/insights/structure/base.py,sha256=sekRrLxB8IvpCzxShPC1L71K36XjsRMJyXzGRz4TcGQ,1101
104
107
  datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py,sha256=NdsXojyRpVv0-jhUWdxg6uvEa0EInT_RR8DYaAsRXKQ,4382
@@ -114,13 +117,16 @@ datapilot/core/platforms/dbt/wrappers/catalog/wrapper.py,sha256=LDjt4pSiqGAmvI5D
114
117
  datapilot/core/platforms/dbt/wrappers/catalog/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
115
118
  datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py,sha256=OHCxG0CJAxOgicGeykoJJ3AnJG8izZvDUGNm7pXajRQ,832
116
119
  datapilot/core/platforms/dbt/wrappers/manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
117
- datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=lKFpmgALDqMEXQldkQ9OFjG51VAfTcZ7rk01e2t39n0,1078
120
+ datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=Q9DG-OEyJAa4ePoVhxG78pehfNotNc9A3xtz171fLhU,1189
118
121
  datapilot/core/platforms/dbt/wrappers/manifest/v10/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
119
122
  datapilot/core/platforms/dbt/wrappers/manifest/v10/schemas.py,sha256=VxqIFQbAL88ObQW1E8wPpiuooCdawdACJUFhgAYBxmo,1523
120
- datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=vch6X_sCF3kw-2NuffYMQlOUSZnYgPLZX6x1CC-8gBI,17360
123
+ datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=7ewyoNPQbjcyNH90cugZh9Nad-cM3bF7XvlZKZ1uEGE,17536
121
124
  datapilot/core/platforms/dbt/wrappers/manifest/v11/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
122
125
  datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py,sha256=Mb0N48czcwQTz5mxQx2QlVINzz50A5FUm0kMJ-Nzs6A,1523
123
- datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=z-KGORZHaQLdLzl8lVw7jYrK6t_rEhQI8horw65j1pY,17360
126
+ datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=DXOxuRHphwKfAr3WPoULteeZGdF8IHlHQu2lm382hrs,17536
127
+ datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
128
+ datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py,sha256=Muu2CVkTKbOtPrOcitysh6bxDrdLVw-Cu6HZT8J7Si8,1248
129
+ datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py,sha256=ONARXJ3MhttSNUGnIc73CAmSiE2pZctHztwKLmZUGqg,17767
124
130
  datapilot/core/platforms/dbt/wrappers/run_results/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
125
131
  datapilot/core/platforms/dbt/wrappers/run_results/run_results.py,sha256=3E_y1gAF491WmXt-Z_Fqhr5BU-kVnzjHpZZv5UpOx-s,1267
126
132
  datapilot/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -133,10 +139,10 @@ datapilot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
133
139
  datapilot/utils/utils.py,sha256=MY8q6ZBJ0hkrTuH7gWMxAlEAQGrajXFMabEhtGtT7sc,11524
134
140
  datapilot/utils/formatting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
135
141
  datapilot/utils/formatting/utils.py,sha256=rAVmIYuldvw9VvCSwG2kMTEgiT7cEconp_F1sAWVyCo,1377
136
- altimate_datapilot_cli-0.0.12.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
137
- altimate_datapilot_cli-0.0.12.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
138
- altimate_datapilot_cli-0.0.12.dist-info/METADATA,sha256=XCKSmeQwcKzUJqlhoa9F3wsArWyp2FYUSZoJow4R2yk,2365
139
- altimate_datapilot_cli-0.0.12.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
140
- altimate_datapilot_cli-0.0.12.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
141
- altimate_datapilot_cli-0.0.12.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
142
- altimate_datapilot_cli-0.0.12.dist-info/RECORD,,
142
+ altimate_datapilot_cli-0.0.14.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
143
+ altimate_datapilot_cli-0.0.14.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
144
+ altimate_datapilot_cli-0.0.14.dist-info/METADATA,sha256=s4DZVAhnfFUP7rj4HVF7H7Ote-5mozGCG8k6UySl5K4,2382
145
+ altimate_datapilot_cli-0.0.14.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
146
+ altimate_datapilot_cli-0.0.14.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
147
+ altimate_datapilot_cli-0.0.14.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
148
+ altimate_datapilot_cli-0.0.14.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.37.1)
2
+ Generator: bdist_wheel (0.44.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
datapilot/__init__.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.0.12"
1
+ __version__ = "0.0.14"
@@ -44,7 +44,7 @@ class APIClient:
44
44
  return response.json()
45
45
 
46
46
  except HTTPError as http_err:
47
- self.logger.error(f"{http_err.response.json()} - Status code: {response.status_code}")
47
+ self.logger.debug(f"HTTP Error: {http_err.response.json()} - Status code: {response.status_code}")
48
48
  except ConnectionError as conn_err:
49
49
  self.logger.error(f"Connection error occurred: {conn_err}")
50
50
  except Timeout as timeout_err:
@@ -84,6 +84,23 @@ class APIClient:
84
84
  endpoint = "/dbt/v3/validate-credentials"
85
85
  return self.get(endpoint)
86
86
 
87
+ def validate_upload_to_integration(self):
88
+ endpoint = "/dbt/v1/validate-permissions"
89
+ return self.get(endpoint)
90
+
87
91
  def start_dbt_ingestion(self, params=None):
88
92
  endpoint = "/dbt/v1/start_dbt_ingestion"
89
93
  return self.post(endpoint, data=params)
94
+
95
+ def get_project_governance_llm_checks(self, params=None):
96
+ endpoint = "/project_governance/checks"
97
+ return self.get(endpoint, params=params)
98
+
99
+ def run_project_governance_llm_checks(self, manifest, catalog, check_names):
100
+ endpoint = "/project_governance/check/run"
101
+ data = {
102
+ "manifest": manifest,
103
+ "catalog": catalog,
104
+ "check_names": check_names,
105
+ }
106
+ return self.post(endpoint, data=data)
@@ -46,6 +46,15 @@ def validate_credentials(
46
46
  return api_client.validate_credentials()
47
47
 
48
48
 
49
+ def validate_permissions(
50
+ token,
51
+ backend_url,
52
+ tenant,
53
+ ) -> Response:
54
+ api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant)
55
+ return api_client.validate_upload_to_integration()
56
+
57
+
49
58
  def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
50
59
  api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
51
60
 
@@ -94,3 +103,24 @@ def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_int
94
103
  "ok": False,
95
104
  "message": "Error starting dbt ingestion worker. ",
96
105
  }
106
+
107
+
108
+ def get_project_governance_llm_checks(
109
+ api_token,
110
+ tenant,
111
+ backend_url,
112
+ ):
113
+ api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
114
+ return api_client.get_project_governance_llm_checks()
115
+
116
+
117
+ def run_project_governance_llm_checks(
118
+ api_token,
119
+ tenant,
120
+ backend_url,
121
+ manifest,
122
+ catalog,
123
+ check_names,
124
+ ):
125
+ api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
126
+ return api_client.run_project_governance_llm_checks(manifest, catalog, check_names)
@@ -1,18 +1,11 @@
1
1
  from abc import abstractmethod
2
- from typing import Optional
3
2
 
4
- from datapilot.core.insights.base.insight import Insight
5
- from datapilot.schemas.sql import Dialect
3
+ from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
6
4
 
7
5
 
8
- class SqlInsight(Insight):
6
+ class SqlInsight(ChecksInsight):
9
7
  NAME = "SqlInsight"
10
8
 
11
- def __init__(self, sql: str, dialect: Optional[Dialect], *args, **kwargs):
12
- self.sql = sql
13
- self.dialect = dialect
14
- super().__init__(*args, **kwargs)
15
-
16
9
  @abstractmethod
17
10
  def generate(self, *args, **kwargs) -> dict:
18
11
  pass
@@ -6,6 +6,7 @@ from datapilot.clients.altimate.utils import check_token_and_instance
6
6
  from datapilot.clients.altimate.utils import onboard_file
7
7
  from datapilot.clients.altimate.utils import start_dbt_ingestion
8
8
  from datapilot.clients.altimate.utils import validate_credentials
9
+ from datapilot.clients.altimate.utils import validate_permissions
9
10
  from datapilot.config.config import load_config
10
11
  from datapilot.core.platforms.dbt.constants import MODEL
11
12
  from datapilot.core.platforms.dbt.constants import PROJECT
@@ -27,6 +28,8 @@ def dbt():
27
28
 
28
29
 
29
30
  @dbt.command("project-health")
31
+ @click.option("--token", required=False, help="Your API token for authentication.")
32
+ @click.option("--instance-name", required=False, help="Your tenant ID.")
30
33
  @click.option(
31
34
  "--manifest-path",
32
35
  required=True,
@@ -48,7 +51,10 @@ def dbt():
48
51
  default=None,
49
52
  help="Selective model testing. Specify one or more models to run tests on.",
50
53
  )
51
- def project_health(manifest_path, catalog_path, config_path=None, select=None):
54
+ @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
55
+ def project_health(
56
+ token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com"
57
+ ):
52
58
  """
53
59
  Validate the DBT project's configuration and structure.
54
60
  :param manifest_path: Path to the DBT manifest file.
@@ -61,7 +67,16 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
61
67
  selected_models = select.split(" ")
62
68
  manifest = load_manifest(manifest_path)
63
69
  catalog = load_catalog(catalog_path) if catalog_path else None
64
- insight_generator = DBTInsightGenerator(manifest=manifest, catalog=catalog, config=config, selected_models=selected_models)
70
+
71
+ insight_generator = DBTInsightGenerator(
72
+ manifest=manifest,
73
+ catalog=catalog,
74
+ config=config,
75
+ selected_models=selected_models,
76
+ token=token,
77
+ instance_name=instance_name,
78
+ backend_url=backend_url,
79
+ )
65
80
  reports = insight_generator.run()
66
81
 
67
82
  package_insights = reports[PROJECT]
@@ -111,6 +126,10 @@ def onboard(
111
126
  click.echo("Error: Invalid credentials.")
112
127
  return
113
128
 
129
+ if not validate_permissions(token, backend_url, instance_name):
130
+ click.echo("Error: You don't have permission to perform this action.")
131
+ return
132
+
114
133
  # This will throw error if manifest file is incorrect
115
134
  try:
116
135
  load_manifest(manifest_path)
@@ -4,6 +4,8 @@ TEST = "test"
4
4
  MODEL = "model"
5
5
  SOURCE = "source"
6
6
 
7
+ LLM = "llm"
8
+
7
9
 
8
10
  PROJECT = "project"
9
11
  SQL = "sql"
@@ -5,11 +5,16 @@ from typing import Dict
5
5
  from typing import List
6
6
  from typing import Optional
7
7
 
8
+ from datapilot.clients.altimate.utils import get_project_governance_llm_checks
9
+ from datapilot.clients.altimate.utils import run_project_governance_llm_checks
10
+ from datapilot.core.platforms.dbt.constants import LLM
8
11
  from datapilot.core.platforms.dbt.constants import MODEL
9
12
  from datapilot.core.platforms.dbt.constants import PROJECT
10
13
  from datapilot.core.platforms.dbt.exceptions import AltimateCLIArgumentError
11
14
  from datapilot.core.platforms.dbt.factory import DBTFactory
12
15
  from datapilot.core.platforms.dbt.insights import INSIGHTS
16
+ from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
17
+ from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
13
18
  from datapilot.core.platforms.dbt.schemas.manifest import Catalog
14
19
  from datapilot.core.platforms.dbt.schemas.manifest import Manifest
15
20
  from datapilot.core.platforms.dbt.utils import get_models
@@ -29,11 +34,19 @@ class DBTInsightGenerator:
29
34
  target: str = "dev",
30
35
  selected_models: Optional[str] = None,
31
36
  selected_model_ids: Optional[List[str]] = None,
37
+ token: Optional[str] = None,
38
+ instance_name: Optional[str] = None,
39
+ backend_url: Optional[str] = None,
32
40
  ):
33
41
  self.run_results_path = run_results_path
34
42
  self.target = target
35
43
  self.env = env
36
44
  self.config = config or {}
45
+ self.token = token
46
+ self.instance_name = instance_name
47
+ self.backend_url = backend_url
48
+ self.manifest = manifest
49
+ self.catalog = catalog
37
50
 
38
51
  self.manifest_wrapper = DBTFactory.get_manifest_wrapper(manifest)
39
52
  self.manifest_present = True
@@ -51,6 +64,7 @@ class DBTInsightGenerator:
51
64
  self.macros = self.manifest_wrapper.get_macros()
52
65
  self.sources = self.manifest_wrapper.get_sources()
53
66
  self.exposures = self.manifest_wrapper.get_exposures()
67
+ self.adapter_type = self.manifest_wrapper.get_adapter_type()
54
68
  self.seeds = self.manifest_wrapper.get_seeds()
55
69
  self.children_map = self.manifest_wrapper.parent_to_child_map(self.nodes)
56
70
  self.tests = self.manifest_wrapper.get_tests()
@@ -85,6 +99,22 @@ class DBTInsightGenerator:
85
99
  return True
86
100
  return False
87
101
 
102
+ def run_llm_checks(self):
103
+ llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url)
104
+ check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])]
105
+ if len(check_names) == 0:
106
+ return {"results": []}
107
+
108
+ llm_check_results = run_project_governance_llm_checks(
109
+ self.token,
110
+ self.instance_name,
111
+ self.backend_url,
112
+ self.manifest.json() if self.manifest else "",
113
+ self.catalog.json() if self.catalog else "",
114
+ check_names,
115
+ )
116
+ return llm_check_results
117
+
88
118
  def run(self):
89
119
  reports = {
90
120
  MODEL: {},
@@ -112,6 +142,7 @@ class DBTInsightGenerator:
112
142
  children_map=self.children_map,
113
143
  tests=self.tests,
114
144
  project_name=self.project_name,
145
+ adapter_type=self.adapter_type,
115
146
  config=self.config,
116
147
  selected_models=self.selected_models,
117
148
  excluded_models=self.excluded_models,
@@ -154,4 +185,42 @@ class DBTInsightGenerator:
154
185
  else:
155
186
  self.logger.info(color_text(f"Skipping insight {insight_class.NAME} as {message}", YELLOW))
156
187
 
188
+ if self.token and self.instance_name and self.backend_url:
189
+ llm_check_results = self.run_llm_checks()
190
+ llm_reports = llm_check_results.get("results", [])
191
+ llm_insights = {}
192
+ for report in llm_reports:
193
+ for answer in report["answer"]:
194
+ location = answer["unique_id"]
195
+ if location not in llm_insights:
196
+ llm_insights[location] = []
197
+ metadata = answer.get("metadata", {})
198
+ metadata["source"] = LLM
199
+ metadata["teammate_check_id"] = report["id"]
200
+ metadata["category"] = report["type"]
201
+ llm_insights[location].append(
202
+ DBTModelInsightResponse(
203
+ insight=DBTInsightResult(
204
+ type="Custom",
205
+ name=report["name"],
206
+ message=answer["message"],
207
+ reason_to_flag=answer["reason_to_flag"],
208
+ recommendation=answer["recommendation"],
209
+ metadata=metadata,
210
+ ),
211
+ severity=answer["severity"],
212
+ path=answer["path"] if answer.get("path") else "",
213
+ original_file_path=answer["original_file_path"] if answer.get("original_file_path") else "",
214
+ package_name=answer["package_name"] if answer.get("package_name") else "",
215
+ unique_id=answer["unique_id"],
216
+ )
217
+ )
218
+
219
+ if llm_insights:
220
+ for key, value in llm_insights.items():
221
+ if key in reports[MODEL]:
222
+ reports[MODEL][key].extend(value)
223
+ else:
224
+ reports[MODEL][key] = value
225
+
157
226
  return reports
@@ -1,18 +1,22 @@
1
1
  from dbt_artifacts_parser.parsers.catalog.catalog_v1 import CatalogV1
2
2
  from dbt_artifacts_parser.parsers.manifest.manifest_v10 import ManifestV10
3
3
  from dbt_artifacts_parser.parsers.manifest.manifest_v11 import ManifestV11
4
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
4
5
 
5
6
  from datapilot.core.platforms.dbt.schemas.manifest import Catalog
6
7
  from datapilot.core.platforms.dbt.schemas.manifest import Manifest
7
8
  from datapilot.core.platforms.dbt.wrappers.catalog.v1.wrapper import CatalogV1Wrapper
8
9
  from datapilot.core.platforms.dbt.wrappers.manifest.v10.wrapper import ManifestV10Wrapper
9
10
  from datapilot.core.platforms.dbt.wrappers.manifest.v11.wrapper import ManifestV11Wrapper
11
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.wrapper import ManifestV12Wrapper
10
12
  from datapilot.exceptions.exceptions import AltimateNotSupportedError
11
13
 
12
14
 
13
15
  class DBTFactory:
14
16
  @classmethod
15
17
  def get_manifest_wrapper(cls, manifest: Manifest):
18
+ if isinstance(manifest, ManifestV12):
19
+ return ManifestV12Wrapper(manifest)
16
20
  if isinstance(manifest, ManifestV11):
17
21
  return ManifestV11Wrapper(manifest)
18
22
  if isinstance(manifest, ManifestV10):
@@ -51,6 +51,7 @@ from datapilot.core.platforms.dbt.insights.modelling.staging_model_dependent_on_
51
51
  from datapilot.core.platforms.dbt.insights.modelling.unused_sources import DBTUnusedSources
52
52
  from datapilot.core.platforms.dbt.insights.performance.chain_view_linking import DBTChainViewLinking
53
53
  from datapilot.core.platforms.dbt.insights.performance.exposure_parent_materializations import DBTExposureParentMaterialization
54
+ from datapilot.core.platforms.dbt.insights.sql.sql_check import SqlCheck
54
55
  from datapilot.core.platforms.dbt.insights.structure.model_directories_structure import DBTModelDirectoryStructure
55
56
  from datapilot.core.platforms.dbt.insights.structure.model_naming_conventions import DBTModelNamingConvention
56
57
  from datapilot.core.platforms.dbt.insights.structure.source_directories_structure import DBTSourceDirectoryStructure
@@ -112,4 +113,5 @@ INSIGHTS = [
112
113
  CheckSourceHasTests,
113
114
  CheckSourceTableHasDescription,
114
115
  CheckSourceTags,
116
+ SqlCheck,
115
117
  ]
@@ -2,6 +2,7 @@ from abc import abstractmethod
2
2
  from typing import ClassVar
3
3
  from typing import Dict
4
4
  from typing import List
5
+ from typing import Optional
5
6
  from typing import Union
6
7
 
7
8
  from datapilot.config.utils import get_insight_config
@@ -33,6 +34,7 @@ class DBTInsight(Insight):
33
34
  macros: Dict[str, AltimateManifestMacroNode],
34
35
  children_map: Dict[str, List[str]],
35
36
  project_name: str,
37
+ adapter_type: Optional[str],
36
38
  selected_models: Union[List[str], None] = None,
37
39
  excluded_models: Union[List[str], None] = None,
38
40
  *args,
@@ -47,6 +49,7 @@ class DBTInsight(Insight):
47
49
  self.seeds = seeds
48
50
  self.children_map = children_map
49
51
  self.project_name = project_name
52
+ self.adapter_type = adapter_type
50
53
  self.selected_models = selected_models
51
54
  self.excluded_models = excluded_models
52
55
  super().__init__(*args, **kwargs)
File without changes
@@ -0,0 +1,23 @@
1
+ from abc import abstractmethod
2
+ from typing import Tuple
3
+
4
+ from datapilot.core.platforms.dbt.insights.base import DBTInsight
5
+
6
+
7
+ class SqlInsight(DBTInsight):
8
+ TYPE = "governance"
9
+
10
+ @abstractmethod
11
+ def generate(self, *args, **kwargs) -> dict:
12
+ pass
13
+
14
+ @classmethod
15
+ def has_all_required_data(cls, has_manifest: bool, **kwargs) -> Tuple[bool, str]:
16
+ """
17
+ Check if all required data is available for the insight to run.
18
+ :param has_manifest: A boolean indicating if manifest is available.
19
+ :return: A boolean indicating if all required data is available.
20
+ """
21
+ if not has_manifest:
22
+ return False, "manifest is required for insight to run."
23
+ return True, ""
@@ -0,0 +1,101 @@
1
+ import inspect
2
+ from typing import List
3
+
4
+ from sqlglot import parse_one
5
+ from sqlglot.optimizer.eliminate_ctes import eliminate_ctes
6
+ from sqlglot.optimizer.eliminate_joins import eliminate_joins
7
+ from sqlglot.optimizer.eliminate_subqueries import eliminate_subqueries
8
+ from sqlglot.optimizer.normalize import normalize
9
+ from sqlglot.optimizer.pushdown_projections import pushdown_projections
10
+ from sqlglot.optimizer.qualify import qualify
11
+ from sqlglot.optimizer.unnest_subqueries import unnest_subqueries
12
+
13
+ from datapilot.core.insights.sql.base.insight import SqlInsight
14
+ from datapilot.core.insights.utils import get_severity
15
+ from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
16
+ from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
17
+
18
+ RULES = (
19
+ pushdown_projections,
20
+ normalize,
21
+ unnest_subqueries,
22
+ eliminate_subqueries,
23
+ eliminate_joins,
24
+ eliminate_ctes,
25
+ )
26
+
27
+
28
+ class SqlCheck(SqlInsight):
29
+ """
30
+ This class identifies DBT models with SQL optimization issues.
31
+ """
32
+
33
+ NAME = "sql optimization issues"
34
+ ALIAS = "check_sql_optimization"
35
+ DESCRIPTION = "Checks if the model has SQL optimization issues. "
36
+ REASON_TO_FLAG = "The query can be optimized."
37
+ FAILURE_MESSAGE = "The query for model `{model_unique_id}` has optimization opportunities:\n{rule_name}. "
38
+ RECOMMENDATION = "Please adapt the query of the model `{model_unique_id}` as in following example:\n{optimized_sql}"
39
+
40
+ def _build_failure_result(self, model_unique_id: str, rule_name: str, optimized_sql: str) -> DBTInsightResult:
41
+ """
42
+ Constructs a failure result for a given model with sql optimization issues.
43
+ :param model_unique_id: The unique id of the dbt model.
44
+ :param rule_name: The rule that generated this failure result.
45
+ :param optimized_sql: The optimized sql.
46
+ :return: An instance of DBTInsightResult containing failure details.
47
+ """
48
+ failure_message = self.FAILURE_MESSAGE.format(model_unique_id=model_unique_id, rule_name=rule_name)
49
+ recommendation = self.RECOMMENDATION.format(model_unique_id=model_unique_id, optimized_sql=optimized_sql)
50
+ return DBTInsightResult(
51
+ type=self.TYPE,
52
+ name=self.NAME,
53
+ message=failure_message,
54
+ recommendation=recommendation,
55
+ reason_to_flag=self.REASON_TO_FLAG,
56
+ metadata={"model_unique_id": model_unique_id, "rule_name": rule_name},
57
+ )
58
+
59
+ def generate(self, *args, **kwargs) -> List[DBTModelInsightResponse]:
60
+ """
61
+ Generates insights for each DBT model in the project, focusing on sql optimization issues.
62
+
63
+ :return: A list of DBTModelInsightResponse objects with insights for each model.
64
+ """
65
+ self.logger.debug("Generating sql insights for DBT models")
66
+ insights = []
67
+
68
+ possible_kwargs = {
69
+ "db": None,
70
+ "catalog": None,
71
+ "dialect": self.adapter_type,
72
+ "isolate_tables": True, # needed for other optimizations to perform well
73
+ "quote_identifiers": False,
74
+ **kwargs,
75
+ }
76
+ for node_id, node in self.nodes.items():
77
+ try:
78
+ compiled_query = node.compiled_code
79
+ if compiled_query:
80
+ parsed_query = parse_one(compiled_query, dialect=self.adapter_type)
81
+ qualified = qualify(parsed_query, **possible_kwargs)
82
+ changed = qualified.copy()
83
+ for rule in RULES:
84
+ original = changed.copy()
85
+ rule_params = inspect.getfullargspec(rule).args
86
+ rule_kwargs = {param: possible_kwargs[param] for param in rule_params if param in possible_kwargs}
87
+ changed = rule(changed, **rule_kwargs)
88
+ if changed.sql() != original.sql():
89
+ insights.append(
90
+ DBTModelInsightResponse(
91
+ unique_id=node_id,
92
+ package_name=node.package_name,
93
+ path=node.original_file_path,
94
+ original_file_path=node.original_file_path,
95
+ insight=self._build_failure_result(node_id, rule.__name__, changed.sql()),
96
+ severity=get_severity(self.config, self.ALIAS, self.DEFAULT_SEVERITY),
97
+ )
98
+ )
99
+ except Exception as e:
100
+ self.logger.error(e)
101
+ return insights
@@ -1,4 +1,5 @@
1
1
  from typing import Dict
2
+ from typing import Optional
2
3
  from typing import Set
3
4
 
4
5
  from dbt_artifacts_parser.parsers.manifest.manifest_v10 import GenericTestNode
@@ -67,6 +68,7 @@ class ManifestV10Wrapper(BaseManifestWrapper):
67
68
  depends_on_macros = node.depends_on.macros if node.depends_on else None
68
69
  compiled_path = node.compiled_path
69
70
  compiled = node.compiled
71
+ compiled_code = node.compiled_code
70
72
  raw_code = node.raw_code
71
73
  language = node.language
72
74
  contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
@@ -381,6 +383,9 @@ class ManifestV10Wrapper(BaseManifestWrapper):
381
383
  seeds[seed.unique_id] = self._get_seed(seed)
382
384
  return seeds
383
385
 
386
+ def get_adapter_type(self) -> Optional[str]:
387
+ return self.manifest.metadata.adapter_type
388
+
384
389
  def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
385
390
  """
386
391
  Current manifest contains information about parents
@@ -1,4 +1,5 @@
1
1
  from typing import Dict
2
+ from typing import Optional
2
3
  from typing import Set
3
4
 
4
5
  from dbt_artifacts_parser.parsers.manifest.manifest_v11 import GenericTestNode
@@ -67,6 +68,7 @@ class ManifestV11Wrapper(BaseManifestWrapper):
67
68
  depends_on_macros = node.depends_on.macros if node.depends_on else None
68
69
  compiled_path = node.compiled_path
69
70
  compiled = node.compiled
71
+ compiled_code = node.compiled_code
70
72
  raw_code = node.raw_code
71
73
  language = node.language
72
74
  contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
@@ -381,6 +383,9 @@ class ManifestV11Wrapper(BaseManifestWrapper):
381
383
  seeds[seed.unique_id] = self._get_seed(seed)
382
384
  return seeds
383
385
 
386
+ def get_adapter_type(self) -> Optional[str]:
387
+ return self.manifest.metadata.adapter_type
388
+
384
389
  def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
385
390
  """
386
391
  Current manifest contains information about parents
@@ -0,0 +1,36 @@
1
+ from typing import Dict
2
+ from typing import Type
3
+ from typing import Union
4
+
5
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Exposures
6
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Macros
7
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node
8
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node1
9
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2
10
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node3
11
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node4
12
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node5
13
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6
14
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node7
15
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Sources
16
+
17
+ from datapilot.core.platforms.dbt.constants import GENERIC
18
+ from datapilot.core.platforms.dbt.constants import SINGULAR
19
+
20
+ ManifestNode = Union[Node, Node1, Node2, Node3, Node4, Node5, Node6, Node7]
21
+
22
+ SourceNode = Sources
23
+
24
+ ExposureNode = Exposures
25
+
26
+ TestNode = Union[Node6, Node2]
27
+
28
+ MacroNode = Macros
29
+
30
+ TEST_TYPE_TO_NODE_MAP: Dict[str, Type] = {
31
+ GENERIC: [Node6],
32
+ SINGULAR: [Node2],
33
+ }
34
+
35
+
36
+ SeedNodeMap = Node
@@ -0,0 +1,414 @@
1
+ from typing import Dict
2
+ from typing import Optional
3
+ from typing import Set
4
+
5
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import ManifestV12
6
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node2
7
+ from dbt_artifacts_parser.parsers.manifest.manifest_v12 import Node6
8
+
9
+ from datapilot.core.platforms.dbt.constants import GENERIC
10
+ from datapilot.core.platforms.dbt.constants import OTHER_TEST_NODE
11
+ from datapilot.core.platforms.dbt.constants import SEED
12
+ from datapilot.core.platforms.dbt.constants import SINGULAR
13
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateDBTContract
14
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateDependsOn
15
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateExposureType
16
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateExternalTable
17
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateFileHash
18
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateFreshnessThreshold
19
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateMacroArgument
20
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestColumnInfo
21
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestExposureNode
22
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestMacroNode
23
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestNode
24
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestSourceNode
25
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestTestNode
26
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateMaturityEnum
27
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateNodeConfig
28
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateOwner
29
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateQuoting
30
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateRefArgs
31
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateResourceType
32
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedConfig
33
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateSeedNode
34
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateSourceConfig
35
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestConfig
36
+ from datapilot.core.platforms.dbt.schemas.manifest import AltimateTestMetadata
37
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import TEST_TYPE_TO_NODE_MAP
38
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import ExposureNode
39
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import MacroNode
40
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import ManifestNode
41
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import SeedNodeMap
42
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import SourceNode
43
+ from datapilot.core.platforms.dbt.wrappers.manifest.v12.schemas import TestNode
44
+ from datapilot.core.platforms.dbt.wrappers.manifest.wrapper import BaseManifestWrapper
45
+
46
+
47
+ class ManifestV12Wrapper(BaseManifestWrapper):
48
+ def __init__(self, manifest: ManifestV12):
49
+ self.manifest = manifest
50
+
51
+ def _get_node(self, node: ManifestNode) -> AltimateManifestNode:
52
+ (
53
+ sources,
54
+ metrics,
55
+ compiled_path,
56
+ compiled,
57
+ compiled_code,
58
+ depends_on_nodes,
59
+ depends_on_macros,
60
+ raw_code,
61
+ language,
62
+ contract,
63
+ ) = ([], [], None, None, None, None, None, "", "", None)
64
+ if node.resource_type != SEED:
65
+ sources = node.sources
66
+ metrics = node.metrics
67
+ depends_on_nodes = node.depends_on.nodes if node.depends_on else None
68
+ depends_on_macros = node.depends_on.macros if node.depends_on else None
69
+ compiled_path = node.compiled_path
70
+ compiled = node.compiled
71
+ compiled_code = node.compiled_code
72
+ raw_code = node.raw_code
73
+ language = node.language
74
+ contract = AltimateDBTContract(**node.contract.__dict__) if node.contract else None
75
+
76
+ return AltimateManifestNode(
77
+ database=node.database,
78
+ schema_name=node.schema_,
79
+ name=node.name,
80
+ resource_type=AltimateResourceType(node.resource_type),
81
+ package_name=node.package_name,
82
+ path=node.path,
83
+ description=node.description,
84
+ original_file_path=node.original_file_path,
85
+ unique_id=node.unique_id,
86
+ fqn=node.fqn,
87
+ alias=node.alias,
88
+ raw_code=raw_code,
89
+ language=language,
90
+ config=AltimateNodeConfig(**node.config.__dict__) if node.config else None,
91
+ checksum=AltimateFileHash(
92
+ name=node.checksum.name if node.checksum else None,
93
+ checksum=node.checksum.checksum if node.checksum else None,
94
+ ),
95
+ columns={
96
+ name: AltimateManifestColumnInfo(
97
+ name=column.name,
98
+ description=column.description,
99
+ meta=column.meta,
100
+ data_type=column.data_type,
101
+ quote=column.quote,
102
+ tags=column.tags,
103
+ )
104
+ for name, column in node.columns.items()
105
+ },
106
+ relation_name=node.relation_name,
107
+ sources=sources,
108
+ metrics=metrics,
109
+ depends_on=AltimateDependsOn(
110
+ nodes=depends_on_nodes,
111
+ macros=depends_on_macros,
112
+ ),
113
+ compiled_path=compiled_path,
114
+ compiled=compiled,
115
+ compiled_code=compiled_code,
116
+ contract=contract,
117
+ meta=node.meta,
118
+ patch_path=node.patch_path,
119
+ )
120
+
121
+ def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
122
+ return AltimateManifestSourceNode(
123
+ database=source.database,
124
+ resource_type=AltimateResourceType(source.resource_type),
125
+ schema_name=source.schema_,
126
+ name=source.name,
127
+ package_name=source.package_name,
128
+ path=source.path,
129
+ original_file_path=source.original_file_path,
130
+ unique_id=source.unique_id,
131
+ fqn=source.fqn,
132
+ source_name=source.source_name,
133
+ source_description=source.source_description,
134
+ loader=source.loader,
135
+ identifier=source.identifier,
136
+ quoting=AltimateQuoting(**source.quoting.dict()) if source.quoting else None,
137
+ loaded_at_field=source.loaded_at_field,
138
+ freshness=AltimateFreshnessThreshold(**source.freshness.dict()) if source.freshness else None,
139
+ external=AltimateExternalTable(**source.external.dict()) if source.external else None,
140
+ description=source.description,
141
+ columns={
142
+ name: AltimateManifestColumnInfo(
143
+ name=column.name,
144
+ description=column.description,
145
+ meta=column.meta,
146
+ data_type=column.data_type,
147
+ quote=column.quote,
148
+ tags=column.tags,
149
+ )
150
+ for name, column in source.columns.items()
151
+ },
152
+ meta=source.meta,
153
+ relation_name=source.relation_name,
154
+ source_meta=source.source_meta,
155
+ tags=source.tags,
156
+ config=AltimateSourceConfig(**source.config.dict()) if source.config else None,
157
+ patch_path=source.patch_path,
158
+ unrendered_config=source.unrendered_config,
159
+ created_at=source.created_at,
160
+ )
161
+
162
+ def _get_macro(self, macro: MacroNode) -> AltimateManifestMacroNode:
163
+ return AltimateManifestMacroNode(
164
+ name=macro.name,
165
+ resource_type=AltimateResourceType(macro.resource_type),
166
+ package_name=macro.package_name,
167
+ path=macro.path,
168
+ original_file_path=macro.original_file_path,
169
+ unique_id=macro.unique_id,
170
+ macro_sql=macro.macro_sql,
171
+ depends_on=(
172
+ AltimateDependsOn(
173
+ macros=macro.depends_on.macros,
174
+ )
175
+ if macro.depends_on
176
+ else None
177
+ ),
178
+ description=macro.description,
179
+ meta=macro.meta,
180
+ docs=macro.docs,
181
+ patch_path=macro.patch_path,
182
+ arguments=[AltimateMacroArgument(**arg.dict()) for arg in macro.arguments] if macro.arguments else None,
183
+ created_at=macro.created_at,
184
+ supported_languages=macro.supported_languages,
185
+ )
186
+
187
+ def _get_exposure(self, exposure: ExposureNode) -> AltimateManifestExposureNode:
188
+ return AltimateManifestExposureNode(
189
+ name=exposure.name,
190
+ resource_type=AltimateResourceType(exposure.resource_type),
191
+ package_name=exposure.package_name,
192
+ path=exposure.path,
193
+ original_file_path=exposure.original_file_path,
194
+ unique_id=exposure.unique_id,
195
+ fqn=exposure.fqn,
196
+ type=AltimateExposureType(exposure.type.value) if exposure.type else None,
197
+ owner=AltimateOwner(**exposure.owner.dict()) if exposure.owner else None,
198
+ description=exposure.description,
199
+ label=exposure.label,
200
+ maturity=AltimateMaturityEnum(exposure.maturity.value) if exposure.maturity else None,
201
+ meta=exposure.meta,
202
+ tags=exposure.tags,
203
+ config=AltimateSourceConfig(**exposure.config.dict()) if exposure.config else None,
204
+ unrendered_config=exposure.unrendered_config,
205
+ url=exposure.url,
206
+ depends_on=(
207
+ AltimateDependsOn(
208
+ nodes=exposure.depends_on.nodes,
209
+ macros=exposure.depends_on.macros,
210
+ )
211
+ if exposure.depends_on
212
+ else None
213
+ ),
214
+ refs=[AltimateRefArgs(**ref.dict()) for ref in exposure.refs] if exposure.refs else None,
215
+ sources=exposure.sources,
216
+ metrics=exposure.metrics,
217
+ created_at=exposure.created_at,
218
+ )
219
+
220
+ def _get_tests(self, test: TestNode) -> AltimateManifestTestNode:
221
+ test_metadata = None
222
+ if isinstance(test, Node6):
223
+ test_type = GENERIC
224
+ test_metadata = AltimateTestMetadata(**test.test_metadata.dict()) if test.test_metadata else None
225
+ elif isinstance(test, Node2):
226
+ test_type = SINGULAR
227
+ else:
228
+ test_type = OTHER_TEST_NODE
229
+ return AltimateManifestTestNode(
230
+ test_metadata=test_metadata,
231
+ test_type=test_type,
232
+ name=test.name,
233
+ resource_type=AltimateResourceType(test.resource_type),
234
+ package_name=test.package_name,
235
+ path=test.path,
236
+ original_file_path=test.original_file_path,
237
+ unique_id=test.unique_id,
238
+ fqn=test.fqn,
239
+ alias=test.alias,
240
+ checksum=(
241
+ AltimateFileHash(
242
+ name=test.checksum.name,
243
+ checksum=test.checksum.checksum,
244
+ )
245
+ if test.checksum
246
+ else None
247
+ ),
248
+ config=AltimateTestConfig(**test.config.dict()) if test.config else None,
249
+ description=test.description,
250
+ tags=test.tags,
251
+ columns=(
252
+ {
253
+ name: AltimateManifestColumnInfo(
254
+ name=column.name,
255
+ description=column.description,
256
+ meta=column.meta,
257
+ data_type=column.data_type,
258
+ quote=column.quote,
259
+ tags=column.tags,
260
+ )
261
+ for name, column in test.columns.items()
262
+ }
263
+ if test.columns
264
+ else None
265
+ ),
266
+ meta=test.meta,
267
+ relation_name=test.relation_name,
268
+ group=test.group,
269
+ raw_code=test.raw_code,
270
+ language=test.language,
271
+ refs=[AltimateRefArgs(**ref.dict()) for ref in test.refs] if test.refs else None,
272
+ sources=test.sources,
273
+ metrics=test.metrics,
274
+ depends_on=(
275
+ AltimateDependsOn(
276
+ nodes=test.depends_on.nodes,
277
+ macros=test.depends_on.macros,
278
+ )
279
+ if test.depends_on
280
+ else None
281
+ ),
282
+ compiled_path=test.compiled_path,
283
+ compiled=test.compiled,
284
+ compiled_code=test.compiled_code,
285
+ )
286
+
287
+ def _get_seed(self, seed: SeedNodeMap) -> AltimateSeedNode:
288
+ return AltimateSeedNode(
289
+ database=seed.database,
290
+ schema_name=seed.schema_,
291
+ name=seed.name,
292
+ resource_type=AltimateResourceType(seed.resource_type),
293
+ package_name=seed.package_name,
294
+ path=seed.path,
295
+ original_file_path=seed.original_file_path,
296
+ unique_id=seed.unique_id,
297
+ fqn=seed.fqn,
298
+ alias=seed.alias,
299
+ checksum=(
300
+ AltimateFileHash(
301
+ name=seed.checksum.name,
302
+ checksum=seed.checksum.checksum,
303
+ )
304
+ if seed.checksum
305
+ else None
306
+ ),
307
+ config=AltimateSeedConfig(**seed.config.dict()) if seed.config else None,
308
+ description=seed.description,
309
+ tags=seed.tags,
310
+ columns=(
311
+ {
312
+ name: AltimateManifestColumnInfo(
313
+ name=column.name,
314
+ description=column.description,
315
+ meta=column.meta,
316
+ data_type=column.data_type,
317
+ quote=column.quote,
318
+ tags=column.tags,
319
+ )
320
+ for name, column in seed.columns.items()
321
+ }
322
+ if seed.columns
323
+ else None
324
+ ),
325
+ meta=seed.meta,
326
+ group=seed.group,
327
+ docs=seed.docs.dict() if seed.docs else None,
328
+ patch_path=seed.patch_path,
329
+ build_path=seed.build_path,
330
+ deferred=False,
331
+ unrendered_config=seed.unrendered_config,
332
+ created_at=seed.created_at,
333
+ config_call_dict=seed.config_call_dict,
334
+ )
335
+
336
+ def get_nodes(
337
+ self,
338
+ ) -> Dict[str, AltimateManifestNode]:
339
+ nodes = {}
340
+ for node in self.manifest.nodes.values():
341
+ if (
342
+ node.resource_type
343
+ in [
344
+ AltimateResourceType.seed.value,
345
+ AltimateResourceType.test.value,
346
+ ]
347
+ or node.package_name != self.get_package()
348
+ ):
349
+ continue
350
+ nodes[node.unique_id] = self._get_node(node)
351
+ return nodes
352
+
353
+ def get_package(self) -> str:
354
+ return self.manifest.metadata.project_name
355
+
356
+ def get_sources(self) -> Dict[str, AltimateManifestSourceNode]:
357
+ sources = {}
358
+ for source in self.manifest.sources.values():
359
+ sources[source.unique_id] = self._get_source(source)
360
+ return sources
361
+
362
+ def get_macros(self) -> Dict[str, AltimateManifestMacroNode]:
363
+ macros = {}
364
+ for macro in self.manifest.macros.values():
365
+ if macro.resource_type == AltimateResourceType.macro.value and macro.package_name == self.get_package():
366
+ macros[macro.unique_id] = self._get_macro(macro)
367
+ return macros
368
+
369
+ def get_exposures(self) -> Dict[str, AltimateManifestExposureNode]:
370
+ exposures = {}
371
+ for exposure in self.manifest.exposures.values():
372
+ exposures[exposure.unique_id] = self._get_exposure(exposure)
373
+ return exposures
374
+
375
+ def get_tests(self, type=None) -> Dict[str, AltimateManifestTestNode]:
376
+ tests = {}
377
+ # Initialize types_union with TestNode
378
+ types = [Node2, Node6]
379
+
380
+ # Add other types to the union if provided
381
+ if type:
382
+ types = TEST_TYPE_TO_NODE_MAP.get(type)
383
+
384
+ for node in self.manifest.nodes.values():
385
+ # Check if the node is a test and of the correct type
386
+ if node.resource_type == AltimateResourceType.test.value:
387
+ if any(isinstance(node, t) for t in types):
388
+ tests[node.unique_id] = self._get_tests(node)
389
+ return tests
390
+
391
+ def get_seeds(self) -> Dict[str, AltimateSeedNode]:
392
+ seeds = {}
393
+ for seed in self.manifest.nodes.values():
394
+ if seed.resource_type == AltimateResourceType.seed.value:
395
+ seeds[seed.unique_id] = self._get_seed(seed)
396
+ return seeds
397
+
398
+ def get_adapter_type(self) -> Optional[str]:
399
+ return self.manifest.metadata.adapter_type
400
+
401
+ def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
402
+ """
403
+ Current manifest contains information about parents
404
+ THis gives an information of node to childre
405
+ :param nodes: A dictionary of nodes in a manifest.
406
+ :return: A dictionary of all the children of a node.
407
+ """
408
+ children_map = {}
409
+ for node_id, node in nodes.items():
410
+ if node_id not in children_map:
411
+ children_map[node_id] = set()
412
+ for parent in node.depends_on.nodes or []:
413
+ children_map.setdefault(parent, set()).add(node_id)
414
+ return children_map
@@ -1,6 +1,7 @@
1
1
  from abc import ABC
2
2
  from abc import abstractmethod
3
3
  from typing import Dict
4
+ from typing import Optional
4
5
  from typing import Set
5
6
 
6
7
  from datapilot.core.platforms.dbt.schemas.manifest import AltimateManifestExposureNode
@@ -26,6 +27,10 @@ class BaseManifestWrapper(ABC):
26
27
  def get_exposures(self) -> Dict[str, AltimateManifestExposureNode]:
27
28
  pass
28
29
 
30
+ @abstractmethod
31
+ def get_adapter_type(self) -> Optional[str]:
32
+ pass
33
+
29
34
  @abstractmethod
30
35
  def parent_to_child_map(self, nodes: Dict[str, AltimateManifestNode]) -> Dict[str, Set[str]]:
31
36
  pass