acryl-datahub 1.0.0.1rc2__py3-none-any.whl → 1.0.0.1rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/METADATA +2541 -2541
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/RECORD +28 -26
- datahub/_version.py +1 -1
- datahub/emitter/rest_emitter.py +2 -2
- datahub/ingestion/graph/client.py +6 -11
- datahub/ingestion/source/common/subtypes.py +1 -1
- datahub/ingestion/source/mlflow.py +19 -1
- datahub/ingestion/source/snowflake/constants.py +1 -0
- datahub/ingestion/source/snowflake/snowflake_config.py +14 -1
- datahub/ingestion/source/snowflake/snowflake_query.py +17 -0
- datahub/ingestion/source/snowflake/snowflake_report.py +3 -0
- datahub/ingestion/source/snowflake/snowflake_schema.py +29 -0
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +112 -42
- datahub/ingestion/source/snowflake/snowflake_utils.py +25 -1
- datahub/ingestion/source/sql/mssql/job_models.py +15 -1
- datahub/ingestion/source/sql/mssql/source.py +8 -4
- datahub/ingestion/source/sql/stored_procedures/__init__.py +0 -0
- datahub/ingestion/source/sql/stored_procedures/base.py +242 -0
- datahub/ingestion/source/sql/{mssql/stored_procedure_lineage.py → stored_procedures/lineage.py} +1 -29
- datahub/ingestion/source/vertexai/vertexai.py +1 -1
- datahub/metadata/schema.avsc +2 -0
- datahub/metadata/schemas/Deprecation.avsc +2 -0
- datahub/metadata/schemas/MetadataChangeEvent.avsc +2 -0
- datahub/sql_parsing/split_statements.py +5 -1
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.0.0.
|
|
1
|
+
acryl_datahub-1.0.0.1rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=ZzXmZcaAOZUvHmN9D5crLfATvvjGO_NN7ItKlRqtLD0,323
|
|
5
5
|
datahub/entrypoints.py,sha256=2TYgHhs3sCxJlojIHjqfxzt3_ImPwPzq4vBtsUuMqu4,8885
|
|
6
6
|
datahub/errors.py,sha256=w6h8b27j9XlmPbTwqpu7-wgiTrXlHzcnUOnJ_iOrwzo,520
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -126,7 +126,7 @@ datahub/emitter/mcp_builder.py,sha256=Q1bX2BthNvZ7ae71XYF6ICoiN8IOqaAd_h3zOct57Q
|
|
|
126
126
|
datahub/emitter/mcp_patch_builder.py,sha256=u7cpW6DkiN7KpLapmMaXgL_FneoN69boxiANbVgMdSI,4564
|
|
127
127
|
datahub/emitter/request_helper.py,sha256=HpI9a9W0TzoVbrs584rF8P8w-IT_iKLmvYmO_6IHhXs,1008
|
|
128
128
|
datahub/emitter/response_helper.py,sha256=lRMvzF-RPHNkN_ONl-N2uJjKh5XtRFrofrdGibVGn2U,4509
|
|
129
|
-
datahub/emitter/rest_emitter.py,sha256=
|
|
129
|
+
datahub/emitter/rest_emitter.py,sha256=4l3_vnOmS8GKTj_HUejg5gJb28QCK0XH_nPVPE5AAp4,29841
|
|
130
130
|
datahub/emitter/serialization_helper.py,sha256=q12Avmf70Vy4ttQGMJoTKlE5EsybMKNg2w3MQeZiHvk,3652
|
|
131
131
|
datahub/emitter/sql_parsing_builder.py,sha256=Cr5imZrm3dYDSCACt5MFscgHCtVbHTD6IjUmsvsKoEs,11991
|
|
132
132
|
datahub/emitter/synchronized_file_emitter.py,sha256=s4ATuxalI4GDAkrZTaGSegxBdvvNPZ9jRSdtElU0kNs,1805
|
|
@@ -171,7 +171,7 @@ datahub/ingestion/glossary/classifier.py,sha256=daLxnVv_JlfB_jBOxH5LrU_xQRndrsGo
|
|
|
171
171
|
datahub/ingestion/glossary/classifier_registry.py,sha256=yFOYLQhDgCLqXYMG3L1BquXafeLcZDcmp8meyw6k9ts,307
|
|
172
172
|
datahub/ingestion/glossary/datahub_classifier.py,sha256=O7wm6gQT1Jf2QSKdWjJQbS5oSzJwplXzfza26Gdq5Mg,7555
|
|
173
173
|
datahub/ingestion/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
174
|
-
datahub/ingestion/graph/client.py,sha256=
|
|
174
|
+
datahub/ingestion/graph/client.py,sha256=rdX2DXqTXyLyS1_qiUzc3zzIE8CFheP2pYi1I68r6Dc,65567
|
|
175
175
|
datahub/ingestion/graph/config.py,sha256=_oha8Je7P80ZmrkZUAaRHyYbdMmTkMI5JkYjEP2Ri1Q,751
|
|
176
176
|
datahub/ingestion/graph/connections.py,sha256=9462L0ZWGKURyypAln25eMPhK3pcufBar9tNDoqspXs,741
|
|
177
177
|
datahub/ingestion/graph/entity_versioning.py,sha256=nrcNz0Qm6kpE6oTu_mrYUQDx14KPspBTc6R9SyFUY6c,6901
|
|
@@ -205,7 +205,7 @@ datahub/ingestion/source/ge_profiling_config.py,sha256=FlWfXoVoayabVXNMB9qETEU0G
|
|
|
205
205
|
datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
|
|
206
206
|
datahub/ingestion/source/ldap.py,sha256=CNr3foofIpoCXu_GGqfcajlQE2qkHr5isYwVcDutdkk,18695
|
|
207
207
|
datahub/ingestion/source/metabase.py,sha256=j8DRV2GvisezidL1JZ5HJLF_hdFdtvaoyDoEdEyh0Ks,32603
|
|
208
|
-
datahub/ingestion/source/mlflow.py,sha256=
|
|
208
|
+
datahub/ingestion/source/mlflow.py,sha256=pWhBEIXxI19-yTFeZU72Pgc3uY9nK0-w2TEpG803HNI,32485
|
|
209
209
|
datahub/ingestion/source/mode.py,sha256=20vWflnRIMWqK8q2Mt2PorMBLnzRAA4bMjcWEyqhTio,64506
|
|
210
210
|
datahub/ingestion/source/mongodb.py,sha256=2C2Cxn8DXL53IbNiywIuKt8UT_EMcPg9f8su-OPSNGU,21237
|
|
211
211
|
datahub/ingestion/source/nifi.py,sha256=w5TPnqPmpotvzSsJROi6nUiHWPUVC6u1g0CzXIE6FNs,56903
|
|
@@ -272,7 +272,7 @@ datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
|
|
|
272
272
|
datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
|
|
273
273
|
datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
|
|
274
274
|
datahub/ingestion/source/common/gcp_credentials_config.py,sha256=_NapGkAqZMbXNClLlmOfANS7U6rChhdthRX9s9iUv9k,2411
|
|
275
|
-
datahub/ingestion/source/common/subtypes.py,sha256=
|
|
275
|
+
datahub/ingestion/source/common/subtypes.py,sha256=nSGKiCD491Bz9MRWzJK3sb10z8vLWP2wwSuSq-L8ELk,3068
|
|
276
276
|
datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
277
277
|
datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
|
|
278
278
|
datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=nxu7osuzqxScPFc-1ODA2M1c_xPNPpRH_SMMU7zKOIE,6212
|
|
@@ -445,25 +445,25 @@ datahub/ingestion/source/sigma/sigma_api.py,sha256=SVvbUs2vjueUdDa-3FzeMsaX5pNpA
|
|
|
445
445
|
datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
446
446
|
datahub/ingestion/source/slack/slack.py,sha256=3N7Yp-u9DvBmo536Z6-pQTrJgSJ3i742GePSgjlBOUU,27616
|
|
447
447
|
datahub/ingestion/source/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
448
|
-
datahub/ingestion/source/snowflake/constants.py,sha256=
|
|
448
|
+
datahub/ingestion/source/snowflake/constants.py,sha256=XCW3vw4JfLn_s8-oXBX6WFNMPOkX5qAQLFX2KqagojQ,2678
|
|
449
449
|
datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
|
|
450
450
|
datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
|
|
451
451
|
datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
|
|
452
|
-
datahub/ingestion/source/snowflake/snowflake_config.py,sha256=
|
|
452
|
+
datahub/ingestion/source/snowflake/snowflake_config.py,sha256=Jm3TW7ed9LYNOZ9egUwkHs2bQv_WlCD6D2QoVxIzxsI,20729
|
|
453
453
|
datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=pEw2O9xoTSIWDiROlkF8k4oj5zBjkqTnynLvut08yhc,17796
|
|
454
454
|
datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
|
|
455
455
|
datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=FBmiONx4EGHWV8RNJT6zHZyntKinPFFyd2oKbTUIbhE,21319
|
|
456
456
|
datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=0DJiSwII6FY34urlBja2FW66NaVvhbBWmG0p7u8Xyrc,7548
|
|
457
457
|
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=gX9E1Z_CemAZsuTDmtvqrxY7vBL2da75j7X8Xwhaf8Y,28441
|
|
458
|
-
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=
|
|
459
|
-
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=
|
|
460
|
-
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=
|
|
461
|
-
datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=
|
|
458
|
+
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=0AMPQ_L7sgQtBizBNEe69-BUM8_wk1m8ystWivwKEMI,40409
|
|
459
|
+
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
|
|
460
|
+
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
|
|
461
|
+
datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=_37-AQyI4uGt4fu-d3v2eAWzQ3uG835ZQxMjFwGYCng,57193
|
|
462
462
|
datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
|
|
463
463
|
datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYhWeK4XafJq3ssFL9kcS03jEeWT4,5506
|
|
464
464
|
datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=1eLYTcgmfzDs9xktMTTE74L5SeNP48Qg3uLr9y-Ez3Y,8733
|
|
465
465
|
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
|
|
466
|
-
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=
|
|
466
|
+
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=22JhQx22kP__ux7mKsXJTFvIYrj9uRym8vKBDsVSKSs,14088
|
|
467
467
|
datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=nAbudDVh9A0kqao3jnIdgBlFNhNk1WIxoU1cofeXkFQ,33905
|
|
468
468
|
datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
469
469
|
datahub/ingestion/source/sql/athena.py,sha256=X4SC1kwKJ3a3T0wGQIlOQxzJKE-LEr-U55QKpRUH4j8,23861
|
|
@@ -492,9 +492,11 @@ datahub/ingestion/source/sql/trino.py,sha256=8viVOu67mhDnsO3LuPSRi1WDR5MLdOXu7HO
|
|
|
492
492
|
datahub/ingestion/source/sql/two_tier_sql_source.py,sha256=YDrGBb5WKVls6qv17QU5foKrf71SydzEltc3WsVAhQc,5732
|
|
493
493
|
datahub/ingestion/source/sql/vertica.py,sha256=_9OgSgIgqBml0av063rb8nACiT3SAmzpw0ouyF91wv8,33382
|
|
494
494
|
datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr5GO9qgjS3R7pLRZSk,87
|
|
495
|
-
datahub/ingestion/source/sql/mssql/job_models.py,sha256=
|
|
496
|
-
datahub/ingestion/source/sql/mssql/source.py,sha256=
|
|
497
|
-
datahub/ingestion/source/sql/
|
|
495
|
+
datahub/ingestion/source/sql/mssql/job_models.py,sha256=nAo3rciu-w2-dXCz6_ekDEbGMEjCMEfh8WvSfXoF2l0,9359
|
|
496
|
+
datahub/ingestion/source/sql/mssql/source.py,sha256=Er0uTAsEOzLg2KVddf22pOdzEAwkOjNdqYCPxeCiHdA,32755
|
|
497
|
+
datahub/ingestion/source/sql/stored_procedures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
498
|
+
datahub/ingestion/source/sql/stored_procedures/base.py,sha256=X4RXYknK8ZPsMgXQFzo2_CsMcXYhDWLm-fLFmZNkZfc,7888
|
|
499
|
+
datahub/ingestion/source/sql/stored_procedures/lineage.py,sha256=9kBoq4JLa4-I3TMAYwChXp76Kag8YBUEIAkaQMplIkY,1844
|
|
498
500
|
datahub/ingestion/source/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
499
501
|
datahub/ingestion/source/state/checkpoint.py,sha256=-fTUZKkY4nHTFqSWZ0jJkkdIu_tWlOjRNhm4FTr4ul4,8860
|
|
500
502
|
datahub/ingestion/source/state/entity_removal_state.py,sha256=zvIsmYg7oiIu2FhecU0VfLBNToUqvKoKyDeiFfkOcyc,6611
|
|
@@ -533,7 +535,7 @@ datahub/ingestion/source/usage/clickhouse_usage.py,sha256=jJ-EUJdS7t4d9RVjLWQQ2e
|
|
|
533
535
|
datahub/ingestion/source/usage/starburst_trino_usage.py,sha256=O3EDEZsXPNdsKGD-jStREA8e4-iTlnqd3ocqtAYFKNA,10544
|
|
534
536
|
datahub/ingestion/source/usage/usage_common.py,sha256=YGszLjmESiUXnpcPfnyQHtoM57HyWsIiLOQd5_sxECg,12238
|
|
535
537
|
datahub/ingestion/source/vertexai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
536
|
-
datahub/ingestion/source/vertexai/vertexai.py,sha256=
|
|
538
|
+
datahub/ingestion/source/vertexai/vertexai.py,sha256=TXTa-Cm1C7xkbTNXNmKr_hi2FDJ9VG-ahrxk9yuxaTg,43635
|
|
537
539
|
datahub/ingestion/source/vertexai/vertexai_config.py,sha256=uMnsv3b6TsPRH26u_JE_v1u0db7ANEAFlVxU5A6ELRM,989
|
|
538
540
|
datahub/ingestion/source/vertexai/vertexai_result_type_utils.py,sha256=fE2l_xXvKONqb4jabl4LtKRBZDnP3koMLJV520wEAMg,2555
|
|
539
541
|
datahub/ingestion/source_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -591,7 +593,7 @@ datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1
|
|
|
591
593
|
datahub/lite/lite_util.py,sha256=Cm6trMTeo0X1fv4nSsW9lC0jqce7Jt-05GhOtIGzsVc,4559
|
|
592
594
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
593
595
|
datahub/metadata/_schema_classes.py,sha256=HvQKAu3eTDt3lkQcoLpBqYlPo0jfBONWCd6T9a1Meck,1011170
|
|
594
|
-
datahub/metadata/schema.avsc,sha256=
|
|
596
|
+
datahub/metadata/schema.avsc,sha256=rX4HzpW5NPL28A2vfu6ESo-hSyQDnK7LUnvasa7dQCE,752628
|
|
595
597
|
datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
|
|
596
598
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
597
599
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -761,7 +763,7 @@ datahub/metadata/schemas/DatasetProfile.avsc,sha256=3ZCU9JD6l2razACp0AY6LLMgnkMT
|
|
|
761
763
|
datahub/metadata/schemas/DatasetProperties.avsc,sha256=DFJn75feqaoQk84zin_o_lqsFFhqkwya5LGC5LLJXbU,4209
|
|
762
764
|
datahub/metadata/schemas/DatasetUpstreamLineage.avsc,sha256=PjAWPbsqwH7FjX2kFDy0dE6ENYOwRynH9vJerWisr2A,5365
|
|
763
765
|
datahub/metadata/schemas/DatasetUsageStatistics.avsc,sha256=JKNy_KlUqr3kt7o1Cu2DiQx3XUG_NQbVFVpeNr4ntCc,6999
|
|
764
|
-
datahub/metadata/schemas/Deprecation.avsc,sha256=
|
|
766
|
+
datahub/metadata/schemas/Deprecation.avsc,sha256=p8SBIuKP3XVGeaBI7rROpLNACuoX8eMLRlZz8lGOYV8,1354
|
|
765
767
|
datahub/metadata/schemas/DisplayProperties.avsc,sha256=MTa_g2s0roxNFFggWU8rslUH3UFe3xe11uUXyh0Go_I,1732
|
|
766
768
|
datahub/metadata/schemas/Documentation.avsc,sha256=9vIJG9B08FFrC3y5c1XVaT5U3c-b5sOAc5foUxMnyCs,4836
|
|
767
769
|
datahub/metadata/schemas/DomainKey.avsc,sha256=TYCcJRWqwbxbQuR5E68pvdeAmfVdYsJuMNhTxVphbqg,676
|
|
@@ -835,7 +837,7 @@ datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKj
|
|
|
835
837
|
datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=Kq2Q9WxZ6nQ8wR4P6wpPCI-J7FwXQyoa10s6BvXtkm8,1110
|
|
836
838
|
datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
|
|
837
839
|
datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
|
|
838
|
-
datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=
|
|
840
|
+
datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=rM6klxSttK-cS083tR0BaVgWVJskjMmhwFbHuV8VKI8,375579
|
|
839
841
|
datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=mpdodpx25E6M1Gq_7slEcPAm-1Es5xPsoqV60HgO7zg,12167
|
|
840
842
|
datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=EMfQrYsuHf1p6UvBjoLtfdTHGe-vGNJaCFEHz8hdKU0,9698
|
|
841
843
|
datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
|
|
@@ -931,7 +933,7 @@ datahub/sql_parsing/_sqlglot_patch.py,sha256=dpTiGj4Jxzvbi6HkYiOTbZ81CTFwEEo6JgY
|
|
|
931
933
|
datahub/sql_parsing/datajob.py,sha256=1X8KpEk-y3_8xJuA_Po27EHZgOcxK9QADI6Om9gSGn0,1751
|
|
932
934
|
datahub/sql_parsing/query_types.py,sha256=FKjDzszZzsrCfYfm7dgD6T_8865qxWl767fdGyHWBh4,2720
|
|
933
935
|
datahub/sql_parsing/schema_resolver.py,sha256=ISuingLcQnOJZkNXBkc73uPwYUbbOtERAjgGhJajDiQ,10782
|
|
934
|
-
datahub/sql_parsing/split_statements.py,sha256=
|
|
936
|
+
datahub/sql_parsing/split_statements.py,sha256=e1vVGJXqJbLPwjpFS6owROxVI6J0wwccyBV_9qxI0mc,9121
|
|
935
937
|
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=A3_0wSxBJSRowEaslptDpBoKO42XXx5UyTEK9PkekIs,70317
|
|
936
938
|
datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
|
|
937
939
|
datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
|
|
@@ -1041,8 +1043,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1041
1043
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1042
1044
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1043
1045
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1044
|
-
acryl_datahub-1.0.0.
|
|
1045
|
-
acryl_datahub-1.0.0.
|
|
1046
|
-
acryl_datahub-1.0.0.
|
|
1047
|
-
acryl_datahub-1.0.0.
|
|
1048
|
-
acryl_datahub-1.0.0.
|
|
1046
|
+
acryl_datahub-1.0.0.1rc3.dist-info/METADATA,sha256=BTS0H-lGX3ONF9A0odwU_U8VbAUTrcd7K56DDTjG9iM,176849
|
|
1047
|
+
acryl_datahub-1.0.0.1rc3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
1048
|
+
acryl_datahub-1.0.0.1rc3.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
|
|
1049
|
+
acryl_datahub-1.0.0.1rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1050
|
+
acryl_datahub-1.0.0.1rc3.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
datahub/emitter/rest_emitter.py
CHANGED
|
@@ -24,9 +24,9 @@ from typing import (
|
|
|
24
24
|
|
|
25
25
|
import pydantic
|
|
26
26
|
import requests
|
|
27
|
-
from deprecated import deprecated
|
|
28
27
|
from requests.adapters import HTTPAdapter, Retry
|
|
29
28
|
from requests.exceptions import HTTPError, RequestException
|
|
29
|
+
from typing_extensions import deprecated
|
|
30
30
|
|
|
31
31
|
from datahub._version import nice_version_name
|
|
32
32
|
from datahub.cli import config_utils
|
|
@@ -597,7 +597,7 @@ class DataHubRestEmitter(Closeable, Emitter):
|
|
|
597
597
|
|
|
598
598
|
return len(mcp_obj_chunks)
|
|
599
599
|
|
|
600
|
-
@deprecated
|
|
600
|
+
@deprecated("Use emit with a datasetUsageStatistics aspect instead")
|
|
601
601
|
def emit_usage(self, usageStats: UsageAggregation) -> None:
|
|
602
602
|
url = f"{self._gms_server}/usageStats?action=batchIngest"
|
|
603
603
|
|
|
@@ -23,9 +23,9 @@ from typing import (
|
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
from avro.schema import RecordSchema
|
|
26
|
-
from deprecated import deprecated
|
|
27
26
|
from pydantic import BaseModel
|
|
28
27
|
from requests.models import HTTPError
|
|
28
|
+
from typing_extensions import deprecated
|
|
29
29
|
|
|
30
30
|
from datahub.cli import config_utils
|
|
31
31
|
from datahub.configuration.common import ConfigModel, GraphError, OperationalError
|
|
@@ -330,7 +330,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
330
330
|
f"Failed to find {aspect_type_name} in response {response_json}"
|
|
331
331
|
)
|
|
332
332
|
|
|
333
|
-
@deprecated(
|
|
333
|
+
@deprecated("Use get_aspect instead which makes aspect string name optional")
|
|
334
334
|
def get_aspect_v2(
|
|
335
335
|
self,
|
|
336
336
|
entity_urn: str,
|
|
@@ -355,7 +355,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
355
355
|
def get_schema_metadata(self, entity_urn: str) -> Optional[SchemaMetadataClass]:
|
|
356
356
|
return self.get_aspect(entity_urn=entity_urn, aspect_type=SchemaMetadataClass)
|
|
357
357
|
|
|
358
|
-
@deprecated(
|
|
358
|
+
@deprecated("Use get_aspect directly.")
|
|
359
359
|
def get_domain_properties(self, entity_urn: str) -> Optional[DomainPropertiesClass]:
|
|
360
360
|
return self.get_aspect(entity_urn=entity_urn, aspect_type=DomainPropertiesClass)
|
|
361
361
|
|
|
@@ -376,7 +376,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
376
376
|
def get_domain(self, entity_urn: str) -> Optional[DomainsClass]:
|
|
377
377
|
return self.get_aspect(entity_urn=entity_urn, aspect_type=DomainsClass)
|
|
378
378
|
|
|
379
|
-
@deprecated(
|
|
379
|
+
@deprecated("Use get_aspect directly.")
|
|
380
380
|
def get_browse_path(self, entity_urn: str) -> Optional[BrowsePathsClass]:
|
|
381
381
|
return self.get_aspect(entity_urn=entity_urn, aspect_type=BrowsePathsClass)
|
|
382
382
|
|
|
@@ -505,7 +505,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
505
505
|
return response.json()
|
|
506
506
|
|
|
507
507
|
@deprecated(
|
|
508
|
-
|
|
508
|
+
"Use get_aspect for a single aspect or get_entity_semityped for a full entity."
|
|
509
509
|
)
|
|
510
510
|
def get_aspects_for_entity(
|
|
511
511
|
self,
|
|
@@ -635,9 +635,6 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
635
635
|
def _aspect_count_endpoint(self):
|
|
636
636
|
return f"{self.config.server}/aspects?action=getCount"
|
|
637
637
|
|
|
638
|
-
# def _session(self) -> Session:
|
|
639
|
-
# return super()._session
|
|
640
|
-
|
|
641
638
|
def get_domain_urn_by_name(self, domain_name: str) -> Optional[str]:
|
|
642
639
|
"""Retrieve a domain urn based on its name. Returns None if there is no match found"""
|
|
643
640
|
|
|
@@ -749,9 +746,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
749
746
|
|
|
750
747
|
assert res["upsertConnection"]["urn"] == urn
|
|
751
748
|
|
|
752
|
-
@deprecated(
|
|
753
|
-
reason='Use get_urns_by_filter(entity_types=["container"], ...) instead'
|
|
754
|
-
)
|
|
749
|
+
@deprecated('Use get_urns_by_filter(entity_types=["container"], ...) instead')
|
|
755
750
|
def get_container_urns_by_filter(
|
|
756
751
|
self,
|
|
757
752
|
env: Optional[str] = None,
|
|
@@ -69,7 +69,7 @@ class FlowContainerSubTypes(StrEnum):
|
|
|
69
69
|
class JobContainerSubTypes(StrEnum):
|
|
70
70
|
NIFI_PROCESS_GROUP = "Process Group"
|
|
71
71
|
MSSQL_JOBSTEP = "Job Step"
|
|
72
|
-
|
|
72
|
+
STORED_PROCEDURE = "Stored Procedure"
|
|
73
73
|
|
|
74
74
|
|
|
75
75
|
class BIAssetSubTypes(StrEnum):
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import os
|
|
2
3
|
import time
|
|
3
4
|
from dataclasses import dataclass
|
|
4
5
|
from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union
|
|
@@ -115,6 +116,13 @@ class MLflowConfig(StatefulIngestionConfigBase, EnvConfigMixin):
|
|
|
115
116
|
default=None, description="Mapping of source type to datahub platform"
|
|
116
117
|
)
|
|
117
118
|
|
|
119
|
+
username: Optional[str] = Field(
|
|
120
|
+
default=None, description="Username for MLflow authentication"
|
|
121
|
+
)
|
|
122
|
+
password: Optional[str] = Field(
|
|
123
|
+
default=None, description="Password for MLflow authentication"
|
|
124
|
+
)
|
|
125
|
+
|
|
118
126
|
|
|
119
127
|
@dataclass
|
|
120
128
|
class MLflowRegisteredModelStageInfo:
|
|
@@ -161,7 +169,17 @@ class MLflowSource(StatefulIngestionSourceBase):
|
|
|
161
169
|
self.ctx = ctx
|
|
162
170
|
self.config = config
|
|
163
171
|
self.report = StaleEntityRemovalSourceReport()
|
|
164
|
-
self.client =
|
|
172
|
+
self.client = self._configure_client()
|
|
173
|
+
|
|
174
|
+
def _configure_client(self) -> MlflowClient:
|
|
175
|
+
if bool(self.config.username) != bool(self.config.password):
|
|
176
|
+
raise ValueError("Both username and password must be set together")
|
|
177
|
+
|
|
178
|
+
if self.config.username and self.config.password:
|
|
179
|
+
os.environ["MLFLOW_TRACKING_USERNAME"] = self.config.username
|
|
180
|
+
os.environ["MLFLOW_TRACKING_PASSWORD"] = self.config.password
|
|
181
|
+
|
|
182
|
+
return MlflowClient(
|
|
165
183
|
tracking_uri=self.config.tracking_uri,
|
|
166
184
|
registry_uri=self.config.registry_uri,
|
|
167
185
|
)
|
|
@@ -100,7 +100,15 @@ class SnowflakeFilterConfig(SQLFilterConfig):
|
|
|
100
100
|
|
|
101
101
|
stream_pattern: AllowDenyPattern = Field(
|
|
102
102
|
default=AllowDenyPattern.allow_all(),
|
|
103
|
-
description="Regex patterns for streams to filter in ingestion.
|
|
103
|
+
description="Regex patterns for streams to filter in ingestion. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
procedure_pattern: AllowDenyPattern = Field(
|
|
107
|
+
default=AllowDenyPattern.allow_all(),
|
|
108
|
+
description="Regex patterns for procedures to filter in ingestion. "
|
|
109
|
+
"Specify regex to match the entire procedure name in database.schema.procedure format. "
|
|
110
|
+
"e.g. to match all procedures starting with customer in Customer database and public schema,"
|
|
111
|
+
" use the regex 'Customer.public.customer.*'",
|
|
104
112
|
)
|
|
105
113
|
|
|
106
114
|
match_fully_qualified_names: bool = Field(
|
|
@@ -284,6 +292,11 @@ class SnowflakeV2Config(
|
|
|
284
292
|
description="If enabled, streams will be ingested as separate entities from tables/views.",
|
|
285
293
|
)
|
|
286
294
|
|
|
295
|
+
include_procedures: bool = Field(
|
|
296
|
+
default=True,
|
|
297
|
+
description="If enabled, procedures will be ingested as pipelines/tasks.",
|
|
298
|
+
)
|
|
299
|
+
|
|
287
300
|
structured_property_pattern: AllowDenyPattern = Field(
|
|
288
301
|
default=AllowDenyPattern.allow_all(),
|
|
289
302
|
description=(
|
|
@@ -164,6 +164,23 @@ class SnowflakeQuery:
|
|
|
164
164
|
and table_type in ('BASE TABLE', 'EXTERNAL TABLE')
|
|
165
165
|
order by table_schema, table_name"""
|
|
166
166
|
|
|
167
|
+
@staticmethod
|
|
168
|
+
def procedures_for_database(db_name: Optional[str]) -> str:
|
|
169
|
+
db_clause = f'"{db_name}".' if db_name is not None else ""
|
|
170
|
+
return f"""
|
|
171
|
+
SELECT procedure_catalog AS "PROCEDURE_CATALOG",
|
|
172
|
+
procedure_schema AS "PROCEDURE_SCHEMA",
|
|
173
|
+
procedure_name AS "PROCEDURE_NAME",
|
|
174
|
+
procedure_language AS "PROCEDURE_LANGUAGE",
|
|
175
|
+
argument_signature AS "ARGUMENT_SIGNATURE",
|
|
176
|
+
data_type AS "PROCEDURE_RETURN_TYPE",
|
|
177
|
+
procedure_definition AS "PROCEDURE_DEFINITION",
|
|
178
|
+
created AS "CREATED",
|
|
179
|
+
last_altered AS "LAST_ALTERED",
|
|
180
|
+
comment AS "COMMENT"
|
|
181
|
+
FROM {db_clause}information_schema.procedures
|
|
182
|
+
order by procedure_schema, procedure_name"""
|
|
183
|
+
|
|
167
184
|
@staticmethod
|
|
168
185
|
def get_all_tags():
|
|
169
186
|
return """
|
|
@@ -105,6 +105,7 @@ class SnowflakeV2Report(
|
|
|
105
105
|
databases_scanned: int = 0
|
|
106
106
|
tags_scanned: int = 0
|
|
107
107
|
streams_scanned: int = 0
|
|
108
|
+
procedures_scanned: int = 0
|
|
108
109
|
|
|
109
110
|
include_usage_stats: bool = False
|
|
110
111
|
include_operational_stats: bool = False
|
|
@@ -163,6 +164,8 @@ class SnowflakeV2Report(
|
|
|
163
164
|
self.tags_scanned += 1
|
|
164
165
|
elif ent_type == "stream":
|
|
165
166
|
self.streams_scanned += 1
|
|
167
|
+
elif ent_type == "procedure":
|
|
168
|
+
self.procedures_scanned += 1
|
|
166
169
|
else:
|
|
167
170
|
raise KeyError(f"Unknown entity {ent_type}.")
|
|
168
171
|
|
|
@@ -14,6 +14,7 @@ from datahub.ingestion.source.snowflake.snowflake_query import (
|
|
|
14
14
|
SnowflakeQuery,
|
|
15
15
|
)
|
|
16
16
|
from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView
|
|
17
|
+
from datahub.ingestion.source.sql.stored_procedures.base import BaseProcedure
|
|
17
18
|
from datahub.utilities.file_backed_collections import FileBackedDict
|
|
18
19
|
from datahub.utilities.prefix_batch_builder import PrefixGroup, build_prefix_batches
|
|
19
20
|
from datahub.utilities.serialized_lru_cache import serialized_lru_cache
|
|
@@ -714,3 +715,31 @@ class SnowflakeDataDictionary(SupportsAsObj):
|
|
|
714
715
|
stream_pagination_marker = stream_name
|
|
715
716
|
|
|
716
717
|
return streams
|
|
718
|
+
|
|
719
|
+
@serialized_lru_cache(maxsize=1)
|
|
720
|
+
def get_procedures_for_database(
|
|
721
|
+
self, db_name: str
|
|
722
|
+
) -> Dict[str, List[BaseProcedure]]:
|
|
723
|
+
procedures: Dict[str, List[BaseProcedure]] = {}
|
|
724
|
+
cur = self.connection.query(
|
|
725
|
+
SnowflakeQuery.procedures_for_database(db_name),
|
|
726
|
+
)
|
|
727
|
+
|
|
728
|
+
for procedure in cur:
|
|
729
|
+
if procedure["PROCEDURE_SCHEMA"] not in procedures:
|
|
730
|
+
procedures[procedure["PROCEDURE_SCHEMA"]] = []
|
|
731
|
+
|
|
732
|
+
procedures[procedure["PROCEDURE_SCHEMA"]].append(
|
|
733
|
+
BaseProcedure(
|
|
734
|
+
name=procedure["PROCEDURE_NAME"],
|
|
735
|
+
language=procedure["PROCEDURE_LANGUAGE"],
|
|
736
|
+
argument_signature=procedure["ARGUMENT_SIGNATURE"],
|
|
737
|
+
return_type=procedure["PROCEDURE_RETURN_TYPE"],
|
|
738
|
+
procedure_definition=procedure["PROCEDURE_DEFINITION"],
|
|
739
|
+
created=procedure["CREATED"],
|
|
740
|
+
last_altered=procedure["LAST_ALTERED"],
|
|
741
|
+
comment=procedure["COMMENT"],
|
|
742
|
+
extra_properties=None,
|
|
743
|
+
)
|
|
744
|
+
)
|
|
745
|
+
return procedures
|