altimate-datapilot-cli 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/METADATA +7 -6
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/RECORD +32 -28
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/WHEEL +1 -1
- datapilot/__init__.py +1 -1
- datapilot/clients/altimate/client.py +18 -1
- datapilot/clients/altimate/utils.py +30 -0
- datapilot/constants.py +7 -0
- datapilot/core/insights/sql/base/insight.py +2 -9
- datapilot/core/platforms/dbt/cli/cli.py +21 -2
- datapilot/core/platforms/dbt/constants.py +2 -0
- datapilot/core/platforms/dbt/executor.py +69 -0
- datapilot/core/platforms/dbt/insights/__init__.py +2 -0
- datapilot/core/platforms/dbt/insights/base.py +3 -0
- datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py +1 -1
- datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py +1 -1
- datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py +1 -1
- datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py +1 -1
- datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py +1 -1
- datapilot/core/platforms/dbt/insights/sql/__init__.py +0 -0
- datapilot/core/platforms/dbt/insights/sql/base.py +23 -0
- datapilot/core/platforms/dbt/insights/sql/sql_check.py +101 -0
- datapilot/core/platforms/dbt/schemas/catalog.py +21 -13
- datapilot/core/platforms/dbt/schemas/manifest.py +41 -27
- datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +82 -64
- datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +93 -75
- datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py +13 -13
- datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py +25 -19
- datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py +5 -0
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/AUTHORS.rst +0 -0
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/LICENSE +0 -0
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/entry_points.txt +0 -0
- {altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/top_level.txt +0 -0
{altimate_datapilot_cli-0.0.13.dist-info → altimate_datapilot_cli-0.0.15.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: altimate-datapilot-cli
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.15
|
4
4
|
Summary: Assistant for Data Teams
|
5
5
|
Home-page: https://github.com/AltimateAI/datapilot-cli
|
6
6
|
Author: Altimate Inc
|
@@ -30,11 +30,12 @@ Classifier: Topic :: Utilities
|
|
30
30
|
Requires-Python: >=3.8
|
31
31
|
License-File: LICENSE
|
32
32
|
License-File: AUTHORS.rst
|
33
|
-
Requires-Dist: click
|
34
|
-
Requires-Dist: dbt-artifacts-parser
|
35
|
-
Requires-Dist: ruamel.yaml
|
36
|
-
Requires-Dist: tabulate
|
37
|
-
Requires-Dist: requests
|
33
|
+
Requires-Dist: click~=8.1.7
|
34
|
+
Requires-Dist: dbt-artifacts-parser~=0.8.1
|
35
|
+
Requires-Dist: ruamel.yaml~=0.18.6
|
36
|
+
Requires-Dist: tabulate~=0.9.0
|
37
|
+
Requires-Dist: requests~=2.31.0
|
38
|
+
Requires-Dist: sqlglot~=25.30.0
|
38
39
|
|
39
40
|
========
|
40
41
|
Overview
|
@@ -1,11 +1,12 @@
|
|
1
|
-
datapilot/__init__.py,sha256=
|
1
|
+
datapilot/__init__.py,sha256=go20U3RCVaJ2N55RnX4tO5rinfUCRV0puFyrHCto8yw,23
|
2
2
|
datapilot/__main__.py,sha256=I9USmeNnK-cAHb6LZfydJC0LeNSE8enieeY55wpR6uw,380
|
3
|
+
datapilot/constants.py,sha256=h0gfZ8qVyTEi0k7NFOQyH4Z1cPU2zb2W7GsoXASgpxg,112
|
3
4
|
datapilot/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
5
|
datapilot/cli/main.py,sha256=VSdqlkCiu8GSG9qQh8q0BzyocsQc4lKWxZAPEsjXF18,181
|
5
6
|
datapilot/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
7
|
datapilot/clients/altimate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
-
datapilot/clients/altimate/client.py,sha256=
|
8
|
-
datapilot/clients/altimate/utils.py,sha256=
|
8
|
+
datapilot/clients/altimate/client.py,sha256=lLIb9wu--5Rk3qpgnmqeyHlWiPmiTIE-nyMXdWodQ5I,3875
|
9
|
+
datapilot/clients/altimate/utils.py,sha256=B94GQgaiBbw4GrLrK8P7r89zzVeYIzr9-8tuJhoN2MY,4302
|
9
10
|
datapilot/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
11
|
datapilot/config/config.py,sha256=kyj53Qsb85V4iGQsX0vSwULOjscMOSFrJDJ3tnagJpo,403
|
11
12
|
datapilot/config/utils.py,sha256=DIAVX-OZ5Lc0Ky_A7dvdbPcD1QSg2DRxZcuaIIZ2rhw,1146
|
@@ -18,23 +19,23 @@ datapilot/core/insights/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
|
|
18
19
|
datapilot/core/insights/base/insight.py,sha256=AshjeLhV7cxwYANfSidDfHZssifTmvymkGHcWkeKipk,883
|
19
20
|
datapilot/core/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
21
|
datapilot/core/insights/sql/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
21
|
-
datapilot/core/insights/sql/base/insight.py,sha256=
|
22
|
+
datapilot/core/insights/sql/base/insight.py,sha256=k8UUn0qrN-QG6NCunPl7Hd6L6kd1X1eUAeGEsyl8v0o,250
|
22
23
|
datapilot/core/insights/sql/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
23
24
|
datapilot/core/insights/sql/static/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
24
25
|
datapilot/core/platforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
25
26
|
datapilot/core/platforms/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
|
-
datapilot/core/platforms/dbt/constants.py,sha256=
|
27
|
+
datapilot/core/platforms/dbt/constants.py,sha256=kW4PEsYWosfDjBEZ9JUhWFtMxKMSNoIn0DNPlJEJuYc,515
|
27
28
|
datapilot/core/platforms/dbt/exceptions.py,sha256=IC5BgcU90gjYYwPcfTlPNtn0_p8fYjavDRMpKZQ0OnY,110
|
28
|
-
datapilot/core/platforms/dbt/executor.py,sha256=
|
29
|
+
datapilot/core/platforms/dbt/executor.py,sha256=4bRNg6yxU1rHu5MDGdq1vO-A0EKS38MNP1hAAkO5h6Y,9992
|
29
30
|
datapilot/core/platforms/dbt/factory.py,sha256=YIQtb-FQQAJsifJ3KiLjjk0WIKTHtEPTNu2MeKHdMG8,1590
|
30
31
|
datapilot/core/platforms/dbt/formatting.py,sha256=bpfa7XmVghTq4WnGDGYC6DruwOwH8YmjFHghoo5cPD8,1638
|
31
32
|
datapilot/core/platforms/dbt/utils.py,sha256=ozFHprR6LTLXQdrGyaRoyIBTua4P1NkP8T7LGgN-9c0,18577
|
32
33
|
datapilot/core/platforms/dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
33
|
-
datapilot/core/platforms/dbt/cli/cli.py,sha256=
|
34
|
+
datapilot/core/platforms/dbt/cli/cli.py,sha256=1phfriG4f5pS1UFwBcZ0uM49iovVpnjdama9lSYcKMU,6266
|
34
35
|
datapilot/core/platforms/dbt/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
36
|
datapilot/core/platforms/dbt/hooks/executor_hook.py,sha256=gSM50vAO7C-f1rdnHogWbqc87aCXPXysZepjp5L2qzw,2966
|
36
|
-
datapilot/core/platforms/dbt/insights/__init__.py,sha256=
|
37
|
-
datapilot/core/platforms/dbt/insights/base.py,sha256=
|
37
|
+
datapilot/core/platforms/dbt/insights/__init__.py,sha256=hk7BAzCTDkY8WNV6L0v-CPn9mrsDyJJusoQxNxGyzAY,7634
|
38
|
+
datapilot/core/platforms/dbt/insights/base.py,sha256=WBp3knDE2GJx-aM-gHhOWWSu1mdAJHzxLd243sLRdRw,5366
|
38
39
|
datapilot/core/platforms/dbt/insights/schema.py,sha256=4nPxEGsgN5sCXQ1BrOesTRphkMKZj1szyk6ckoz49eg,2657
|
39
40
|
datapilot/core/platforms/dbt/insights/utils.py,sha256=2btHwxzxEm7cjoLSzzbnZ3q3caI-58NJeWM0_TouJhI,267
|
40
41
|
datapilot/core/platforms/dbt/insights/checks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -44,8 +45,8 @@ datapilot/core/platforms/dbt/insights/checks/check_column_name_contract.py,sha25
|
|
44
45
|
datapilot/core/platforms/dbt/insights/checks/check_macro_args_have_desc.py,sha256=Px-gBR0dx9z-xLOcp2ctDuKMRKK4ai554woMpeBFGFs,3185
|
45
46
|
datapilot/core/platforms/dbt/insights/checks/check_macro_has_desc.py,sha256=UrMYyZpi97gIBvH7leu5z27BowaFfPl0tOI4dqMSJmQ,2802
|
46
47
|
datapilot/core/platforms/dbt/insights/checks/check_model_has_all_columns.py,sha256=LULWYrGHrXiEcFfv5VxbrQ3quw4h5mLn3Sd_pG4ayug,4277
|
47
|
-
datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py,sha256
|
48
|
-
datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py,sha256=
|
48
|
+
datapilot/core/platforms/dbt/insights/checks/check_model_has_labels_keys.py,sha256=-cxWasRTJbZBhCd4JtTeygEJ90-m7v1YyhUzn0Jdjmw,5189
|
49
|
+
datapilot/core/platforms/dbt/insights/checks/check_model_has_meta_keys.py,sha256=QClFNsYlxy8nvmf8LyJENk8kT624Cas9l78qSXjs868,5009
|
49
50
|
datapilot/core/platforms/dbt/insights/checks/check_model_has_properties_file.py,sha256=hyl51rqSGmbj5DINtL5Dvex80SGSmN2o-qf_U6eaNMg,2968
|
50
51
|
datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_group.py,sha256=d2Csu6EAapFx6eIXuCaCdNllbsjIBVab5QmlIv-SqGM,5706
|
51
52
|
datapilot/core/platforms/dbt/insights/checks/check_model_has_tests_by_name.py,sha256=bu87eQ5yZYZHtmdMEoIUA1Ynv8Tw3UGwCYKB_1_AXDI,5187
|
@@ -59,10 +60,10 @@ datapilot/core/platforms/dbt/insights/checks/check_model_tags.py,sha256=-268fA2k
|
|
59
60
|
datapilot/core/platforms/dbt/insights/checks/check_source_childs.py,sha256=L8W66Hauwjg98OCE3NFfTzXIbmT0XehD2TysiaPe-Eo,4254
|
60
61
|
datapilot/core/platforms/dbt/insights/checks/check_source_columns_have_desc.py,sha256=UzohMY6PGH_d-VCVJxvAbDI_DqeQqukmCpKU9W9fk_g,4453
|
61
62
|
datapilot/core/platforms/dbt/insights/checks/check_source_has_all_columns.py,sha256=AkAyNas0_dl7EN-hZitN33nTaAM11zsNvlGJjLnZOd0,4650
|
62
|
-
datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py,sha256=
|
63
|
-
datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py,sha256=
|
63
|
+
datapilot/core/platforms/dbt/insights/checks/check_source_has_freshness.py,sha256=1KP6g_-98hyjs3dTibFJFgBm97x_hJCeIYHdUR2HKDs,4117
|
64
|
+
datapilot/core/platforms/dbt/insights/checks/check_source_has_labels_keys.py,sha256=UMEd2inyjQMNk7scnn196yoPtQYfOynS4nVGoyGpsqc,5262
|
64
65
|
datapilot/core/platforms/dbt/insights/checks/check_source_has_loader.py,sha256=d3KixZeAVTlcKl8t-ap2n7pZU0qXDBeXYzjf813GkEE,2772
|
65
|
-
datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py,sha256=
|
66
|
+
datapilot/core/platforms/dbt/insights/checks/check_source_has_meta_keys.py,sha256=e4RNkMgGLRxOzqidMRZWKQgDdlFlB7LhHpoWlB5cJrQ,5239
|
66
67
|
datapilot/core/platforms/dbt/insights/checks/check_source_has_tests.py,sha256=M7WWX1ud49FnInS_htvmNa_8gIjVuhk9tIuq7Q-4P5E,3711
|
67
68
|
datapilot/core/platforms/dbt/insights/checks/check_source_has_tests_by_group.py,sha256=tvflQTxgdejTecDBspD6PdLLDHIWXY-_4uOzPWWqh2w,5550
|
68
69
|
datapilot/core/platforms/dbt/insights/checks/check_source_has_tests_by_name.py,sha256=wDNYB0ZGx_hV8DxXewR1TtZ_HnozZN1Dyk0YOlGUoEs,5159
|
@@ -99,6 +100,9 @@ datapilot/core/platforms/dbt/insights/performance/__init__.py,sha256=47DEQpj8HBS
|
|
99
100
|
datapilot/core/platforms/dbt/insights/performance/base.py,sha256=4EE7WmwlqMQSmBcpYeZrellvGU87F81cTL5wxBV5QSo,829
|
100
101
|
datapilot/core/platforms/dbt/insights/performance/chain_view_linking.py,sha256=gqL4qqNAybOTYhc9VGlrbQQJ1PkDtBHFKrcQi_XRpTM,3627
|
101
102
|
datapilot/core/platforms/dbt/insights/performance/exposure_parent_materializations.py,sha256=bX7yqls9eMHShXeNdZ8J3IefM3Gp-4D9OGXAYfrKHEs,4689
|
103
|
+
datapilot/core/platforms/dbt/insights/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
104
|
+
datapilot/core/platforms/dbt/insights/sql/base.py,sha256=dpmHtX3rg1_ewECcsdiVkndftK0EBOnvOfsLw2TUg8U,733
|
105
|
+
datapilot/core/platforms/dbt/insights/sql/sql_check.py,sha256=a84YLtjJFtF1NRBbWueSdAwp6NNi6ZU8_ScbwLfpnfk,4652
|
102
106
|
datapilot/core/platforms/dbt/insights/structure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
103
107
|
datapilot/core/platforms/dbt/insights/structure/base.py,sha256=sekRrLxB8IvpCzxShPC1L71K36XjsRMJyXzGRz4TcGQ,1101
|
104
108
|
datapilot/core/platforms/dbt/insights/structure/model_directories_structure.py,sha256=NdsXojyRpVv0-jhUWdxg6uvEa0EInT_RR8DYaAsRXKQ,4382
|
@@ -106,24 +110,24 @@ datapilot/core/platforms/dbt/insights/structure/model_naming_conventions.py,sha2
|
|
106
110
|
datapilot/core/platforms/dbt/insights/structure/source_directories_structure.py,sha256=K-xTjYltuzcHH51s6PbFYcrnqpSCYluTbrzW1JT_vIU,3675
|
107
111
|
datapilot/core/platforms/dbt/insights/structure/test_directory_structure.py,sha256=XL6YQgWmf397W2OwBmT7L8zUN93M1bOZBTp4PlngqlI,3484
|
108
112
|
datapilot/core/platforms/dbt/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
109
|
-
datapilot/core/platforms/dbt/schemas/catalog.py,sha256=
|
110
|
-
datapilot/core/platforms/dbt/schemas/manifest.py,sha256=
|
113
|
+
datapilot/core/platforms/dbt/schemas/catalog.py,sha256=t8qDw2QTI6e3TMeVt_apw9L-WAe8JyPyZS6nf61m4rc,2023
|
114
|
+
datapilot/core/platforms/dbt/schemas/manifest.py,sha256=zBr6-NgpKx06hQnw5yHmBwR0lOl2L8zyNISvN-eWD6g,14401
|
111
115
|
datapilot/core/platforms/dbt/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
112
116
|
datapilot/core/platforms/dbt/wrappers/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
113
117
|
datapilot/core/platforms/dbt/wrappers/catalog/wrapper.py,sha256=LDjt4pSiqGAmvI5DSPzEubwq13N1MKv9xakLNhG-BWY,196
|
114
118
|
datapilot/core/platforms/dbt/wrappers/catalog/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
115
119
|
datapilot/core/platforms/dbt/wrappers/catalog/v1/wrapper.py,sha256=OHCxG0CJAxOgicGeykoJJ3AnJG8izZvDUGNm7pXajRQ,832
|
116
120
|
datapilot/core/platforms/dbt/wrappers/manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
117
|
-
datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=
|
121
|
+
datapilot/core/platforms/dbt/wrappers/manifest/wrapper.py,sha256=Q9DG-OEyJAa4ePoVhxG78pehfNotNc9A3xtz171fLhU,1189
|
118
122
|
datapilot/core/platforms/dbt/wrappers/manifest/v10/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
119
123
|
datapilot/core/platforms/dbt/wrappers/manifest/v10/schemas.py,sha256=VxqIFQbAL88ObQW1E8wPpiuooCdawdACJUFhgAYBxmo,1523
|
120
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=
|
124
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py,sha256=jXF_HDoBMR5r7IPNcL3YGAvY0ZEFpTmda0uNiElbJbY,18057
|
121
125
|
datapilot/core/platforms/dbt/wrappers/manifest/v11/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
122
126
|
datapilot/core/platforms/dbt/wrappers/manifest/v11/schemas.py,sha256=Mb0N48czcwQTz5mxQx2QlVINzz50A5FUm0kMJ-Nzs6A,1523
|
123
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=
|
127
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py,sha256=vuGFupQHyAJokWkgdzXmZd67S04nRkH89fEYCXeJWa0,17991
|
124
128
|
datapilot/core/platforms/dbt/wrappers/manifest/v12/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
125
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py,sha256=
|
126
|
-
datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py,sha256=
|
129
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v12/schemas.py,sha256=2rMEQ_BCcjiibfrlDKwgCE8vFX_6BzSInpilpVTWrNE,1269
|
130
|
+
datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py,sha256=lAQY7rAI2luQep4sPogi1swVzcAZ_z5VzGwIl2XpKe0,17926
|
127
131
|
datapilot/core/platforms/dbt/wrappers/run_results/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
128
132
|
datapilot/core/platforms/dbt/wrappers/run_results/run_results.py,sha256=3E_y1gAF491WmXt-Z_Fqhr5BU-kVnzjHpZZv5UpOx-s,1267
|
129
133
|
datapilot/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -136,10 +140,10 @@ datapilot/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
136
140
|
datapilot/utils/utils.py,sha256=MY8q6ZBJ0hkrTuH7gWMxAlEAQGrajXFMabEhtGtT7sc,11524
|
137
141
|
datapilot/utils/formatting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
138
142
|
datapilot/utils/formatting/utils.py,sha256=rAVmIYuldvw9VvCSwG2kMTEgiT7cEconp_F1sAWVyCo,1377
|
139
|
-
altimate_datapilot_cli-0.0.
|
140
|
-
altimate_datapilot_cli-0.0.
|
141
|
-
altimate_datapilot_cli-0.0.
|
142
|
-
altimate_datapilot_cli-0.0.
|
143
|
-
altimate_datapilot_cli-0.0.
|
144
|
-
altimate_datapilot_cli-0.0.
|
145
|
-
altimate_datapilot_cli-0.0.
|
143
|
+
altimate_datapilot_cli-0.0.15.dist-info/AUTHORS.rst,sha256=S4H4zw_v3GVyz5_55jF5Gf_YNG3s5Y0VgbQaEov9PFk,50
|
144
|
+
altimate_datapilot_cli-0.0.15.dist-info/LICENSE,sha256=Mf7VqpsmU2QR5_s2Cb_ZeeMB2Q9KW7YXJENZPFZRK1k,1100
|
145
|
+
altimate_datapilot_cli-0.0.15.dist-info/METADATA,sha256=WKXEoLY1k78gtF1S4mw3IYJZUu2Butum_gkDHf_n0dI,2382
|
146
|
+
altimate_datapilot_cli-0.0.15.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
147
|
+
altimate_datapilot_cli-0.0.15.dist-info/entry_points.txt,sha256=0zwgKxN40RLVB5jSmlJz7IH_FBqRtpFdbrdZn-xuQIY,141
|
148
|
+
altimate_datapilot_cli-0.0.15.dist-info/top_level.txt,sha256=gAOFOdwB00vcxv74y4M1J-nQtPvEatU8-mYViEBcToo,10
|
149
|
+
altimate_datapilot_cli-0.0.15.dist-info/RECORD,,
|
datapilot/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.0.
|
1
|
+
__version__ = "0.0.15"
|
@@ -44,7 +44,7 @@ class APIClient:
|
|
44
44
|
return response.json()
|
45
45
|
|
46
46
|
except HTTPError as http_err:
|
47
|
-
self.logger.
|
47
|
+
self.logger.debug(f"HTTP Error: {http_err.response.json()} - Status code: {response.status_code}")
|
48
48
|
except ConnectionError as conn_err:
|
49
49
|
self.logger.error(f"Connection error occurred: {conn_err}")
|
50
50
|
except Timeout as timeout_err:
|
@@ -84,6 +84,23 @@ class APIClient:
|
|
84
84
|
endpoint = "/dbt/v3/validate-credentials"
|
85
85
|
return self.get(endpoint)
|
86
86
|
|
87
|
+
def validate_upload_to_integration(self):
|
88
|
+
endpoint = "/dbt/v1/validate-permissions"
|
89
|
+
return self.get(endpoint)
|
90
|
+
|
87
91
|
def start_dbt_ingestion(self, params=None):
|
88
92
|
endpoint = "/dbt/v1/start_dbt_ingestion"
|
89
93
|
return self.post(endpoint, data=params)
|
94
|
+
|
95
|
+
def get_project_governance_llm_checks(self, params=None):
|
96
|
+
endpoint = "/project_governance/checks"
|
97
|
+
return self.get(endpoint, params=params)
|
98
|
+
|
99
|
+
def run_project_governance_llm_checks(self, manifest, catalog, check_names):
|
100
|
+
endpoint = "/project_governance/check/run"
|
101
|
+
data = {
|
102
|
+
"manifest": manifest,
|
103
|
+
"catalog": catalog,
|
104
|
+
"check_names": check_names,
|
105
|
+
}
|
106
|
+
return self.post(endpoint, data=data)
|
@@ -46,6 +46,15 @@ def validate_credentials(
|
|
46
46
|
return api_client.validate_credentials()
|
47
47
|
|
48
48
|
|
49
|
+
def validate_permissions(
|
50
|
+
token,
|
51
|
+
backend_url,
|
52
|
+
tenant,
|
53
|
+
) -> Response:
|
54
|
+
api_client = APIClient(api_token=token, base_url=backend_url, tenant=tenant)
|
55
|
+
return api_client.validate_upload_to_integration()
|
56
|
+
|
57
|
+
|
49
58
|
def onboard_file(api_token, tenant, dbt_core_integration_id, dbt_core_integration_environment, file_type, file_path, backend_url) -> Dict:
|
50
59
|
api_client = APIClient(api_token, base_url=backend_url, tenant=tenant)
|
51
60
|
|
@@ -94,3 +103,24 @@ def start_dbt_ingestion(api_token, tenant, dbt_core_integration_id, dbt_core_int
|
|
94
103
|
"ok": False,
|
95
104
|
"message": "Error starting dbt ingestion worker. ",
|
96
105
|
}
|
106
|
+
|
107
|
+
|
108
|
+
def get_project_governance_llm_checks(
|
109
|
+
api_token,
|
110
|
+
tenant,
|
111
|
+
backend_url,
|
112
|
+
):
|
113
|
+
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
|
114
|
+
return api_client.get_project_governance_llm_checks()
|
115
|
+
|
116
|
+
|
117
|
+
def run_project_governance_llm_checks(
|
118
|
+
api_token,
|
119
|
+
tenant,
|
120
|
+
backend_url,
|
121
|
+
manifest,
|
122
|
+
catalog,
|
123
|
+
check_names,
|
124
|
+
):
|
125
|
+
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
|
126
|
+
return api_client.run_project_governance_llm_checks(manifest, catalog, check_names)
|
datapilot/constants.py
ADDED
@@ -1,18 +1,11 @@
|
|
1
1
|
from abc import abstractmethod
|
2
|
-
from typing import Optional
|
3
2
|
|
4
|
-
from datapilot.core.insights.base
|
5
|
-
from datapilot.schemas.sql import Dialect
|
3
|
+
from datapilot.core.platforms.dbt.insights.checks.base import ChecksInsight
|
6
4
|
|
7
5
|
|
8
|
-
class SqlInsight(
|
6
|
+
class SqlInsight(ChecksInsight):
|
9
7
|
NAME = "SqlInsight"
|
10
8
|
|
11
|
-
def __init__(self, sql: str, dialect: Optional[Dialect], *args, **kwargs):
|
12
|
-
self.sql = sql
|
13
|
-
self.dialect = dialect
|
14
|
-
super().__init__(*args, **kwargs)
|
15
|
-
|
16
9
|
@abstractmethod
|
17
10
|
def generate(self, *args, **kwargs) -> dict:
|
18
11
|
pass
|
@@ -6,6 +6,7 @@ from datapilot.clients.altimate.utils import check_token_and_instance
|
|
6
6
|
from datapilot.clients.altimate.utils import onboard_file
|
7
7
|
from datapilot.clients.altimate.utils import start_dbt_ingestion
|
8
8
|
from datapilot.clients.altimate.utils import validate_credentials
|
9
|
+
from datapilot.clients.altimate.utils import validate_permissions
|
9
10
|
from datapilot.config.config import load_config
|
10
11
|
from datapilot.core.platforms.dbt.constants import MODEL
|
11
12
|
from datapilot.core.platforms.dbt.constants import PROJECT
|
@@ -27,6 +28,8 @@ def dbt():
|
|
27
28
|
|
28
29
|
|
29
30
|
@dbt.command("project-health")
|
31
|
+
@click.option("--token", required=False, help="Your API token for authentication.")
|
32
|
+
@click.option("--instance-name", required=False, help="Your tenant ID.")
|
30
33
|
@click.option(
|
31
34
|
"--manifest-path",
|
32
35
|
required=True,
|
@@ -48,7 +51,10 @@ def dbt():
|
|
48
51
|
default=None,
|
49
52
|
help="Selective model testing. Specify one or more models to run tests on.",
|
50
53
|
)
|
51
|
-
|
54
|
+
@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
|
55
|
+
def project_health(
|
56
|
+
token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com"
|
57
|
+
):
|
52
58
|
"""
|
53
59
|
Validate the DBT project's configuration and structure.
|
54
60
|
:param manifest_path: Path to the DBT manifest file.
|
@@ -61,7 +67,16 @@ def project_health(manifest_path, catalog_path, config_path=None, select=None):
|
|
61
67
|
selected_models = select.split(" ")
|
62
68
|
manifest = load_manifest(manifest_path)
|
63
69
|
catalog = load_catalog(catalog_path) if catalog_path else None
|
64
|
-
|
70
|
+
|
71
|
+
insight_generator = DBTInsightGenerator(
|
72
|
+
manifest=manifest,
|
73
|
+
catalog=catalog,
|
74
|
+
config=config,
|
75
|
+
selected_models=selected_models,
|
76
|
+
token=token,
|
77
|
+
instance_name=instance_name,
|
78
|
+
backend_url=backend_url,
|
79
|
+
)
|
65
80
|
reports = insight_generator.run()
|
66
81
|
|
67
82
|
package_insights = reports[PROJECT]
|
@@ -111,6 +126,10 @@ def onboard(
|
|
111
126
|
click.echo("Error: Invalid credentials.")
|
112
127
|
return
|
113
128
|
|
129
|
+
if not validate_permissions(token, backend_url, instance_name):
|
130
|
+
click.echo("Error: You don't have permission to perform this action.")
|
131
|
+
return
|
132
|
+
|
114
133
|
# This will throw error if manifest file is incorrect
|
115
134
|
try:
|
116
135
|
load_manifest(manifest_path)
|
@@ -5,11 +5,16 @@ from typing import Dict
|
|
5
5
|
from typing import List
|
6
6
|
from typing import Optional
|
7
7
|
|
8
|
+
from datapilot.clients.altimate.utils import get_project_governance_llm_checks
|
9
|
+
from datapilot.clients.altimate.utils import run_project_governance_llm_checks
|
10
|
+
from datapilot.core.platforms.dbt.constants import LLM
|
8
11
|
from datapilot.core.platforms.dbt.constants import MODEL
|
9
12
|
from datapilot.core.platforms.dbt.constants import PROJECT
|
10
13
|
from datapilot.core.platforms.dbt.exceptions import AltimateCLIArgumentError
|
11
14
|
from datapilot.core.platforms.dbt.factory import DBTFactory
|
12
15
|
from datapilot.core.platforms.dbt.insights import INSIGHTS
|
16
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTInsightResult
|
17
|
+
from datapilot.core.platforms.dbt.insights.schema import DBTModelInsightResponse
|
13
18
|
from datapilot.core.platforms.dbt.schemas.manifest import Catalog
|
14
19
|
from datapilot.core.platforms.dbt.schemas.manifest import Manifest
|
15
20
|
from datapilot.core.platforms.dbt.utils import get_models
|
@@ -29,11 +34,19 @@ class DBTInsightGenerator:
|
|
29
34
|
target: str = "dev",
|
30
35
|
selected_models: Optional[str] = None,
|
31
36
|
selected_model_ids: Optional[List[str]] = None,
|
37
|
+
token: Optional[str] = None,
|
38
|
+
instance_name: Optional[str] = None,
|
39
|
+
backend_url: Optional[str] = None,
|
32
40
|
):
|
33
41
|
self.run_results_path = run_results_path
|
34
42
|
self.target = target
|
35
43
|
self.env = env
|
36
44
|
self.config = config or {}
|
45
|
+
self.token = token
|
46
|
+
self.instance_name = instance_name
|
47
|
+
self.backend_url = backend_url
|
48
|
+
self.manifest = manifest
|
49
|
+
self.catalog = catalog
|
37
50
|
|
38
51
|
self.manifest_wrapper = DBTFactory.get_manifest_wrapper(manifest)
|
39
52
|
self.manifest_present = True
|
@@ -51,6 +64,7 @@ class DBTInsightGenerator:
|
|
51
64
|
self.macros = self.manifest_wrapper.get_macros()
|
52
65
|
self.sources = self.manifest_wrapper.get_sources()
|
53
66
|
self.exposures = self.manifest_wrapper.get_exposures()
|
67
|
+
self.adapter_type = self.manifest_wrapper.get_adapter_type()
|
54
68
|
self.seeds = self.manifest_wrapper.get_seeds()
|
55
69
|
self.children_map = self.manifest_wrapper.parent_to_child_map(self.nodes)
|
56
70
|
self.tests = self.manifest_wrapper.get_tests()
|
@@ -85,6 +99,22 @@ class DBTInsightGenerator:
|
|
85
99
|
return True
|
86
100
|
return False
|
87
101
|
|
102
|
+
def run_llm_checks(self):
|
103
|
+
llm_checks = get_project_governance_llm_checks(self.token, self.instance_name, self.backend_url)
|
104
|
+
check_names = [check["name"] for check in llm_checks if check["alias"] not in self.config.get("disabled_insights", [])]
|
105
|
+
if len(check_names) == 0:
|
106
|
+
return {"results": []}
|
107
|
+
|
108
|
+
llm_check_results = run_project_governance_llm_checks(
|
109
|
+
self.token,
|
110
|
+
self.instance_name,
|
111
|
+
self.backend_url,
|
112
|
+
self.manifest.json() if self.manifest else "",
|
113
|
+
self.catalog.json() if self.catalog else "",
|
114
|
+
check_names,
|
115
|
+
)
|
116
|
+
return llm_check_results
|
117
|
+
|
88
118
|
def run(self):
|
89
119
|
reports = {
|
90
120
|
MODEL: {},
|
@@ -112,6 +142,7 @@ class DBTInsightGenerator:
|
|
112
142
|
children_map=self.children_map,
|
113
143
|
tests=self.tests,
|
114
144
|
project_name=self.project_name,
|
145
|
+
adapter_type=self.adapter_type,
|
115
146
|
config=self.config,
|
116
147
|
selected_models=self.selected_models,
|
117
148
|
excluded_models=self.excluded_models,
|
@@ -154,4 +185,42 @@ class DBTInsightGenerator:
|
|
154
185
|
else:
|
155
186
|
self.logger.info(color_text(f"Skipping insight {insight_class.NAME} as {message}", YELLOW))
|
156
187
|
|
188
|
+
if self.token and self.instance_name and self.backend_url:
|
189
|
+
llm_check_results = self.run_llm_checks()
|
190
|
+
llm_reports = llm_check_results.get("results", [])
|
191
|
+
llm_insights = {}
|
192
|
+
for report in llm_reports:
|
193
|
+
for answer in report["answer"]:
|
194
|
+
location = answer["unique_id"]
|
195
|
+
if location not in llm_insights:
|
196
|
+
llm_insights[location] = []
|
197
|
+
metadata = answer.get("metadata", {})
|
198
|
+
metadata["source"] = LLM
|
199
|
+
metadata["teammate_check_id"] = report["id"]
|
200
|
+
metadata["category"] = report["type"]
|
201
|
+
llm_insights[location].append(
|
202
|
+
DBTModelInsightResponse(
|
203
|
+
insight=DBTInsightResult(
|
204
|
+
type="Custom",
|
205
|
+
name=report["name"],
|
206
|
+
message=answer["message"],
|
207
|
+
reason_to_flag=answer["reason_to_flag"],
|
208
|
+
recommendation=answer["recommendation"],
|
209
|
+
metadata=metadata,
|
210
|
+
),
|
211
|
+
severity=answer["severity"],
|
212
|
+
path=answer["path"] if answer.get("path") else "",
|
213
|
+
original_file_path=answer["original_file_path"] if answer.get("original_file_path") else "",
|
214
|
+
package_name=answer["package_name"] if answer.get("package_name") else "",
|
215
|
+
unique_id=answer["unique_id"],
|
216
|
+
)
|
217
|
+
)
|
218
|
+
|
219
|
+
if llm_insights:
|
220
|
+
for key, value in llm_insights.items():
|
221
|
+
if key in reports[MODEL]:
|
222
|
+
reports[MODEL][key].extend(value)
|
223
|
+
else:
|
224
|
+
reports[MODEL][key] = value
|
225
|
+
|
157
226
|
return reports
|
@@ -51,6 +51,7 @@ from datapilot.core.platforms.dbt.insights.modelling.staging_model_dependent_on_
|
|
51
51
|
from datapilot.core.platforms.dbt.insights.modelling.unused_sources import DBTUnusedSources
|
52
52
|
from datapilot.core.platforms.dbt.insights.performance.chain_view_linking import DBTChainViewLinking
|
53
53
|
from datapilot.core.platforms.dbt.insights.performance.exposure_parent_materializations import DBTExposureParentMaterialization
|
54
|
+
from datapilot.core.platforms.dbt.insights.sql.sql_check import SqlCheck
|
54
55
|
from datapilot.core.platforms.dbt.insights.structure.model_directories_structure import DBTModelDirectoryStructure
|
55
56
|
from datapilot.core.platforms.dbt.insights.structure.model_naming_conventions import DBTModelNamingConvention
|
56
57
|
from datapilot.core.platforms.dbt.insights.structure.source_directories_structure import DBTSourceDirectoryStructure
|
@@ -112,4 +113,5 @@ INSIGHTS = [
|
|
112
113
|
CheckSourceHasTests,
|
113
114
|
CheckSourceTableHasDescription,
|
114
115
|
CheckSourceTags,
|
116
|
+
SqlCheck,
|
115
117
|
]
|
@@ -2,6 +2,7 @@ from abc import abstractmethod
|
|
2
2
|
from typing import ClassVar
|
3
3
|
from typing import Dict
|
4
4
|
from typing import List
|
5
|
+
from typing import Optional
|
5
6
|
from typing import Union
|
6
7
|
|
7
8
|
from datapilot.config.utils import get_insight_config
|
@@ -33,6 +34,7 @@ class DBTInsight(Insight):
|
|
33
34
|
macros: Dict[str, AltimateManifestMacroNode],
|
34
35
|
children_map: Dict[str, List[str]],
|
35
36
|
project_name: str,
|
37
|
+
adapter_type: Optional[str],
|
36
38
|
selected_models: Union[List[str], None] = None,
|
37
39
|
excluded_models: Union[List[str], None] = None,
|
38
40
|
*args,
|
@@ -47,6 +49,7 @@ class DBTInsight(Insight):
|
|
47
49
|
self.seeds = seeds
|
48
50
|
self.children_map = children_map
|
49
51
|
self.project_name = project_name
|
52
|
+
self.adapter_type = adapter_type
|
50
53
|
self.selected_models = selected_models
|
51
54
|
self.excluded_models = excluded_models
|
52
55
|
super().__init__(*args, **kwargs)
|
@@ -76,7 +76,7 @@ class CheckModelHasLabelsKeys(ChecksInsight):
|
|
76
76
|
def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]:
|
77
77
|
status_code = 0
|
78
78
|
missing_keys = set(self.labels_keys) - set(self.get_node(node_id).label)
|
79
|
-
config = self.get_node(node_id).config.
|
79
|
+
config = self.get_node(node_id).config.model_dump() if self.get_node(node_id).config else {}
|
80
80
|
labels = config.get("labels", {})
|
81
81
|
label_keys = set(labels.keys())
|
82
82
|
extra_keys = set()
|
@@ -72,7 +72,7 @@ class CheckModelHasMetaKeys(ChecksInsight):
|
|
72
72
|
def _check_meta_keys(self, node_id) -> Tuple[int, Set[str], Set[str]]:
|
73
73
|
status_code = 0
|
74
74
|
model = self.get_node(node_id)
|
75
|
-
meta = model.meta
|
75
|
+
meta = model.meta if model.meta else {}
|
76
76
|
model_meta_keys = set(meta.keys())
|
77
77
|
missing_keys = None
|
78
78
|
extra_keys = None
|
@@ -61,7 +61,7 @@ class CheckSourceHasFreshness(ChecksInsight):
|
|
61
61
|
|
62
62
|
def _check_source_has_freshness(self, source_id: str) -> List[str]:
|
63
63
|
source = self.get_node(source_id)
|
64
|
-
freshness = source.freshness.
|
64
|
+
freshness = source.freshness.model_dump() if source.freshness else {}
|
65
65
|
|
66
66
|
if not freshness:
|
67
67
|
return False
|
@@ -74,7 +74,7 @@ class CheckSourceHasLabelsKeys(ChecksInsight):
|
|
74
74
|
def _check_labels_keys(self, node_id) -> Tuple[int, Set[str]]:
|
75
75
|
status_code = 0
|
76
76
|
missing_keys = set(self.labels_keys) - set(self.get_node(node_id).label)
|
77
|
-
config = self.get_node(node_id).config.
|
77
|
+
config = self.get_node(node_id).config.model_dump() if self.get_node(node_id).config else {}
|
78
78
|
labels = config.get("labels", {})
|
79
79
|
label_keys = set(labels.keys())
|
80
80
|
extra_keys = set()
|
@@ -81,7 +81,7 @@ class CheckSourceHasMetaKeys(ChecksInsight):
|
|
81
81
|
def _check_source_has_meta_keys(self, source_unique_id: str):
|
82
82
|
status_code = 0
|
83
83
|
model = self.get_node(source_unique_id)
|
84
|
-
meta = model.meta
|
84
|
+
meta = model.meta if model.meta else {}
|
85
85
|
model_meta_keys = set(meta.keys())
|
86
86
|
missing_keys = None
|
87
87
|
extra_keys = None
|
File without changes
|
@@ -0,0 +1,23 @@
|
|
1
|
+
from abc import abstractmethod
|
2
|
+
from typing import Tuple
|
3
|
+
|
4
|
+
from datapilot.core.platforms.dbt.insights.base import DBTInsight
|
5
|
+
|
6
|
+
|
7
|
+
class SqlInsight(DBTInsight):
|
8
|
+
TYPE = "governance"
|
9
|
+
|
10
|
+
@abstractmethod
|
11
|
+
def generate(self, *args, **kwargs) -> dict:
|
12
|
+
pass
|
13
|
+
|
14
|
+
@classmethod
|
15
|
+
def has_all_required_data(cls, has_manifest: bool, **kwargs) -> Tuple[bool, str]:
|
16
|
+
"""
|
17
|
+
Check if all required data is available for the insight to run.
|
18
|
+
:param has_manifest: A boolean indicating if manifest is available.
|
19
|
+
:return: A boolean indicating if all required data is available.
|
20
|
+
"""
|
21
|
+
if not has_manifest:
|
22
|
+
return False, "manifest is required for insight to run."
|
23
|
+
return True, ""
|