acryl-datahub 1.0.0.1rc2__py3-none-any.whl → 1.0.0.1rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (28) hide show
  1. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/METADATA +2541 -2541
  2. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/RECORD +28 -26
  3. datahub/_version.py +1 -1
  4. datahub/emitter/rest_emitter.py +2 -2
  5. datahub/ingestion/graph/client.py +6 -11
  6. datahub/ingestion/source/common/subtypes.py +1 -1
  7. datahub/ingestion/source/mlflow.py +19 -1
  8. datahub/ingestion/source/snowflake/constants.py +1 -0
  9. datahub/ingestion/source/snowflake/snowflake_config.py +14 -1
  10. datahub/ingestion/source/snowflake/snowflake_query.py +17 -0
  11. datahub/ingestion/source/snowflake/snowflake_report.py +3 -0
  12. datahub/ingestion/source/snowflake/snowflake_schema.py +29 -0
  13. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +112 -42
  14. datahub/ingestion/source/snowflake/snowflake_utils.py +25 -1
  15. datahub/ingestion/source/sql/mssql/job_models.py +15 -1
  16. datahub/ingestion/source/sql/mssql/source.py +8 -4
  17. datahub/ingestion/source/sql/stored_procedures/__init__.py +0 -0
  18. datahub/ingestion/source/sql/stored_procedures/base.py +242 -0
  19. datahub/ingestion/source/sql/{mssql/stored_procedure_lineage.py → stored_procedures/lineage.py} +1 -29
  20. datahub/ingestion/source/vertexai/vertexai.py +1 -1
  21. datahub/metadata/schema.avsc +2 -0
  22. datahub/metadata/schemas/Deprecation.avsc +2 -0
  23. datahub/metadata/schemas/MetadataChangeEvent.avsc +2 -0
  24. datahub/sql_parsing/split_statements.py +5 -1
  25. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/WHEEL +0 -0
  26. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/entry_points.txt +0 -0
  27. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/licenses/LICENSE +0 -0
  28. {acryl_datahub-1.0.0.1rc2.dist-info → acryl_datahub-1.0.0.1rc3.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
- acryl_datahub-1.0.0.1rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
1
+ acryl_datahub-1.0.0.1rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
2
2
  datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
3
3
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
4
- datahub/_version.py,sha256=PvsZOwOq_x83_b1FnzbszoDGkE2YlEwlQD9lHFDgAvM,323
4
+ datahub/_version.py,sha256=ZzXmZcaAOZUvHmN9D5crLfATvvjGO_NN7ItKlRqtLD0,323
5
5
  datahub/entrypoints.py,sha256=2TYgHhs3sCxJlojIHjqfxzt3_ImPwPzq4vBtsUuMqu4,8885
6
6
  datahub/errors.py,sha256=w6h8b27j9XlmPbTwqpu7-wgiTrXlHzcnUOnJ_iOrwzo,520
7
7
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -126,7 +126,7 @@ datahub/emitter/mcp_builder.py,sha256=Q1bX2BthNvZ7ae71XYF6ICoiN8IOqaAd_h3zOct57Q
126
126
  datahub/emitter/mcp_patch_builder.py,sha256=u7cpW6DkiN7KpLapmMaXgL_FneoN69boxiANbVgMdSI,4564
127
127
  datahub/emitter/request_helper.py,sha256=HpI9a9W0TzoVbrs584rF8P8w-IT_iKLmvYmO_6IHhXs,1008
128
128
  datahub/emitter/response_helper.py,sha256=lRMvzF-RPHNkN_ONl-N2uJjKh5XtRFrofrdGibVGn2U,4509
129
- datahub/emitter/rest_emitter.py,sha256=LyJuTZicSxzyLlwCyVzecjuyFDH0HkvQmSh037OLBc8,29777
129
+ datahub/emitter/rest_emitter.py,sha256=4l3_vnOmS8GKTj_HUejg5gJb28QCK0XH_nPVPE5AAp4,29841
130
130
  datahub/emitter/serialization_helper.py,sha256=q12Avmf70Vy4ttQGMJoTKlE5EsybMKNg2w3MQeZiHvk,3652
131
131
  datahub/emitter/sql_parsing_builder.py,sha256=Cr5imZrm3dYDSCACt5MFscgHCtVbHTD6IjUmsvsKoEs,11991
132
132
  datahub/emitter/synchronized_file_emitter.py,sha256=s4ATuxalI4GDAkrZTaGSegxBdvvNPZ9jRSdtElU0kNs,1805
@@ -171,7 +171,7 @@ datahub/ingestion/glossary/classifier.py,sha256=daLxnVv_JlfB_jBOxH5LrU_xQRndrsGo
171
171
  datahub/ingestion/glossary/classifier_registry.py,sha256=yFOYLQhDgCLqXYMG3L1BquXafeLcZDcmp8meyw6k9ts,307
172
172
  datahub/ingestion/glossary/datahub_classifier.py,sha256=O7wm6gQT1Jf2QSKdWjJQbS5oSzJwplXzfza26Gdq5Mg,7555
173
173
  datahub/ingestion/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
174
- datahub/ingestion/graph/client.py,sha256=4F-IMnz4DwIdLPL5yXMRqqlB3uonN2OHOUIKKHByS7o,65680
174
+ datahub/ingestion/graph/client.py,sha256=rdX2DXqTXyLyS1_qiUzc3zzIE8CFheP2pYi1I68r6Dc,65567
175
175
  datahub/ingestion/graph/config.py,sha256=_oha8Je7P80ZmrkZUAaRHyYbdMmTkMI5JkYjEP2Ri1Q,751
176
176
  datahub/ingestion/graph/connections.py,sha256=9462L0ZWGKURyypAln25eMPhK3pcufBar9tNDoqspXs,741
177
177
  datahub/ingestion/graph/entity_versioning.py,sha256=nrcNz0Qm6kpE6oTu_mrYUQDx14KPspBTc6R9SyFUY6c,6901
@@ -205,7 +205,7 @@ datahub/ingestion/source/ge_profiling_config.py,sha256=FlWfXoVoayabVXNMB9qETEU0G
205
205
  datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
206
206
  datahub/ingestion/source/ldap.py,sha256=CNr3foofIpoCXu_GGqfcajlQE2qkHr5isYwVcDutdkk,18695
207
207
  datahub/ingestion/source/metabase.py,sha256=j8DRV2GvisezidL1JZ5HJLF_hdFdtvaoyDoEdEyh0Ks,32603
208
- datahub/ingestion/source/mlflow.py,sha256=YhbNEeUnZR44LUlj-s9YcSz9BTcIW7W3ouS_9Sy-G5w,31799
208
+ datahub/ingestion/source/mlflow.py,sha256=pWhBEIXxI19-yTFeZU72Pgc3uY9nK0-w2TEpG803HNI,32485
209
209
  datahub/ingestion/source/mode.py,sha256=20vWflnRIMWqK8q2Mt2PorMBLnzRAA4bMjcWEyqhTio,64506
210
210
  datahub/ingestion/source/mongodb.py,sha256=2C2Cxn8DXL53IbNiywIuKt8UT_EMcPg9f8su-OPSNGU,21237
211
211
  datahub/ingestion/source/nifi.py,sha256=w5TPnqPmpotvzSsJROi6nUiHWPUVC6u1g0CzXIE6FNs,56903
@@ -272,7 +272,7 @@ datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
272
272
  datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
273
273
  datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
274
274
  datahub/ingestion/source/common/gcp_credentials_config.py,sha256=_NapGkAqZMbXNClLlmOfANS7U6rChhdthRX9s9iUv9k,2411
275
- datahub/ingestion/source/common/subtypes.py,sha256=5pHq4awOu7kUFjBde7ftjlWklRmJNYIUfAoKAvrX56g,3074
275
+ datahub/ingestion/source/common/subtypes.py,sha256=nSGKiCD491Bz9MRWzJK3sb10z8vLWP2wwSuSq-L8ELk,3068
276
276
  datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
277
277
  datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
278
278
  datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=nxu7osuzqxScPFc-1ODA2M1c_xPNPpRH_SMMU7zKOIE,6212
@@ -445,25 +445,25 @@ datahub/ingestion/source/sigma/sigma_api.py,sha256=SVvbUs2vjueUdDa-3FzeMsaX5pNpA
445
445
  datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
446
446
  datahub/ingestion/source/slack/slack.py,sha256=3N7Yp-u9DvBmo536Z6-pQTrJgSJ3i742GePSgjlBOUU,27616
447
447
  datahub/ingestion/source/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
448
- datahub/ingestion/source/snowflake/constants.py,sha256=SZzQTO6spPeHfV55tIbg7SL2ecsQF_z169HJ8u2G3Hk,2650
448
+ datahub/ingestion/source/snowflake/constants.py,sha256=XCW3vw4JfLn_s8-oXBX6WFNMPOkX5qAQLFX2KqagojQ,2678
449
449
  datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
450
450
  datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
451
451
  datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
452
- datahub/ingestion/source/snowflake/snowflake_config.py,sha256=c9LE7nl6tkz7P9tc4EkSKphextW5pejLzdP3qS_iL1s,20196
452
+ datahub/ingestion/source/snowflake/snowflake_config.py,sha256=Jm3TW7ed9LYNOZ9egUwkHs2bQv_WlCD6D2QoVxIzxsI,20729
453
453
  datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=pEw2O9xoTSIWDiROlkF8k4oj5zBjkqTnynLvut08yhc,17796
454
454
  datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
455
455
  datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=FBmiONx4EGHWV8RNJT6zHZyntKinPFFyd2oKbTUIbhE,21319
456
456
  datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=0DJiSwII6FY34urlBja2FW66NaVvhbBWmG0p7u8Xyrc,7548
457
457
  datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=gX9E1Z_CemAZsuTDmtvqrxY7vBL2da75j7X8Xwhaf8Y,28441
458
- datahub/ingestion/source/snowflake/snowflake_query.py,sha256=qz_rhRMNCXxHd23bePbb3YxhFgN7eRpV4s6g58hQ5bU,39678
459
- datahub/ingestion/source/snowflake/snowflake_report.py,sha256=ahea-bwpW6T0iDehGo0Qq_J7wKxPkV61aYHm8bGwDqo,6651
460
- datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=qkGgk6WdKSPThFjexXHrxUPYiVtzDk2MbGX3b281A4c,26044
461
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=Yba6OIWYtukAFMoNtEtX2BXWwJee17Dl58DUyK0myho,54530
458
+ datahub/ingestion/source/snowflake/snowflake_query.py,sha256=0AMPQ_L7sgQtBizBNEe69-BUM8_wk1m8ystWivwKEMI,40409
459
+ datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
460
+ datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
461
+ datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=_37-AQyI4uGt4fu-d3v2eAWzQ3uG835ZQxMjFwGYCng,57193
462
462
  datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
463
463
  datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYhWeK4XafJq3ssFL9kcS03jEeWT4,5506
464
464
  datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=1eLYTcgmfzDs9xktMTTE74L5SeNP48Qg3uLr9y-Ez3Y,8733
465
465
  datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
466
- datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=EmYb2FEcdLwei92atRBQ3iKH7av4YBZCIFTgPmLo0Ng,13092
466
+ datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=22JhQx22kP__ux7mKsXJTFvIYrj9uRym8vKBDsVSKSs,14088
467
467
  datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=nAbudDVh9A0kqao3jnIdgBlFNhNk1WIxoU1cofeXkFQ,33905
468
468
  datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
469
469
  datahub/ingestion/source/sql/athena.py,sha256=X4SC1kwKJ3a3T0wGQIlOQxzJKE-LEr-U55QKpRUH4j8,23861
@@ -492,9 +492,11 @@ datahub/ingestion/source/sql/trino.py,sha256=8viVOu67mhDnsO3LuPSRi1WDR5MLdOXu7HO
492
492
  datahub/ingestion/source/sql/two_tier_sql_source.py,sha256=YDrGBb5WKVls6qv17QU5foKrf71SydzEltc3WsVAhQc,5732
493
493
  datahub/ingestion/source/sql/vertica.py,sha256=_9OgSgIgqBml0av063rb8nACiT3SAmzpw0ouyF91wv8,33382
494
494
  datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr5GO9qgjS3R7pLRZSk,87
495
- datahub/ingestion/source/sql/mssql/job_models.py,sha256=5-QQv8w-KnyNq_y-VmSC_K5sr0VoZhfYW6Aasd-z2LY,8901
496
- datahub/ingestion/source/sql/mssql/source.py,sha256=QxgUWL-aSjTXmqZPD_7811MsrDsgW_I9_oMimomdE0A,32593
497
- datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py,sha256=RpnvKPalAAaOD_eUg8bZ4VkGTSeLFWuy0mefwc4s3x8,2837
495
+ datahub/ingestion/source/sql/mssql/job_models.py,sha256=nAo3rciu-w2-dXCz6_ekDEbGMEjCMEfh8WvSfXoF2l0,9359
496
+ datahub/ingestion/source/sql/mssql/source.py,sha256=Er0uTAsEOzLg2KVddf22pOdzEAwkOjNdqYCPxeCiHdA,32755
497
+ datahub/ingestion/source/sql/stored_procedures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
498
+ datahub/ingestion/source/sql/stored_procedures/base.py,sha256=X4RXYknK8ZPsMgXQFzo2_CsMcXYhDWLm-fLFmZNkZfc,7888
499
+ datahub/ingestion/source/sql/stored_procedures/lineage.py,sha256=9kBoq4JLa4-I3TMAYwChXp76Kag8YBUEIAkaQMplIkY,1844
498
500
  datahub/ingestion/source/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
499
501
  datahub/ingestion/source/state/checkpoint.py,sha256=-fTUZKkY4nHTFqSWZ0jJkkdIu_tWlOjRNhm4FTr4ul4,8860
500
502
  datahub/ingestion/source/state/entity_removal_state.py,sha256=zvIsmYg7oiIu2FhecU0VfLBNToUqvKoKyDeiFfkOcyc,6611
@@ -533,7 +535,7 @@ datahub/ingestion/source/usage/clickhouse_usage.py,sha256=jJ-EUJdS7t4d9RVjLWQQ2e
533
535
  datahub/ingestion/source/usage/starburst_trino_usage.py,sha256=O3EDEZsXPNdsKGD-jStREA8e4-iTlnqd3ocqtAYFKNA,10544
534
536
  datahub/ingestion/source/usage/usage_common.py,sha256=YGszLjmESiUXnpcPfnyQHtoM57HyWsIiLOQd5_sxECg,12238
535
537
  datahub/ingestion/source/vertexai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
536
- datahub/ingestion/source/vertexai/vertexai.py,sha256=V_6vdpczbwOsxNStmoi0A6RVC8skL0AerGIfBaDhNmk,43654
538
+ datahub/ingestion/source/vertexai/vertexai.py,sha256=TXTa-Cm1C7xkbTNXNmKr_hi2FDJ9VG-ahrxk9yuxaTg,43635
537
539
  datahub/ingestion/source/vertexai/vertexai_config.py,sha256=uMnsv3b6TsPRH26u_JE_v1u0db7ANEAFlVxU5A6ELRM,989
538
540
  datahub/ingestion/source/vertexai/vertexai_result_type_utils.py,sha256=fE2l_xXvKONqb4jabl4LtKRBZDnP3koMLJV520wEAMg,2555
539
541
  datahub/ingestion/source_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -591,7 +593,7 @@ datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1
591
593
  datahub/lite/lite_util.py,sha256=Cm6trMTeo0X1fv4nSsW9lC0jqce7Jt-05GhOtIGzsVc,4559
592
594
  datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
593
595
  datahub/metadata/_schema_classes.py,sha256=HvQKAu3eTDt3lkQcoLpBqYlPo0jfBONWCd6T9a1Meck,1011170
594
- datahub/metadata/schema.avsc,sha256=kFapzCAb4IbAfpp5lrCeshc9vQfPsXlOg3_BeRxS4yQ,752514
596
+ datahub/metadata/schema.avsc,sha256=rX4HzpW5NPL28A2vfu6ESo-hSyQDnK7LUnvasa7dQCE,752628
595
597
  datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
596
598
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
597
599
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -761,7 +763,7 @@ datahub/metadata/schemas/DatasetProfile.avsc,sha256=3ZCU9JD6l2razACp0AY6LLMgnkMT
761
763
  datahub/metadata/schemas/DatasetProperties.avsc,sha256=DFJn75feqaoQk84zin_o_lqsFFhqkwya5LGC5LLJXbU,4209
762
764
  datahub/metadata/schemas/DatasetUpstreamLineage.avsc,sha256=PjAWPbsqwH7FjX2kFDy0dE6ENYOwRynH9vJerWisr2A,5365
763
765
  datahub/metadata/schemas/DatasetUsageStatistics.avsc,sha256=JKNy_KlUqr3kt7o1Cu2DiQx3XUG_NQbVFVpeNr4ntCc,6999
764
- datahub/metadata/schemas/Deprecation.avsc,sha256=SmbTlMB9fujdMBjYEQkzaU4XJzwM1gD6E8L2zoL1b4Q,1280
766
+ datahub/metadata/schemas/Deprecation.avsc,sha256=p8SBIuKP3XVGeaBI7rROpLNACuoX8eMLRlZz8lGOYV8,1354
765
767
  datahub/metadata/schemas/DisplayProperties.avsc,sha256=MTa_g2s0roxNFFggWU8rslUH3UFe3xe11uUXyh0Go_I,1732
766
768
  datahub/metadata/schemas/Documentation.avsc,sha256=9vIJG9B08FFrC3y5c1XVaT5U3c-b5sOAc5foUxMnyCs,4836
767
769
  datahub/metadata/schemas/DomainKey.avsc,sha256=TYCcJRWqwbxbQuR5E68pvdeAmfVdYsJuMNhTxVphbqg,676
@@ -835,7 +837,7 @@ datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKj
835
837
  datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=Kq2Q9WxZ6nQ8wR4P6wpPCI-J7FwXQyoa10s6BvXtkm8,1110
836
838
  datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
837
839
  datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
838
- datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=fg44ZwOegINWo93q7l62_GG4g7koEamX-3JYlC9as0I,375469
840
+ datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=rM6klxSttK-cS083tR0BaVgWVJskjMmhwFbHuV8VKI8,375579
839
841
  datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=mpdodpx25E6M1Gq_7slEcPAm-1Es5xPsoqV60HgO7zg,12167
840
842
  datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=EMfQrYsuHf1p6UvBjoLtfdTHGe-vGNJaCFEHz8hdKU0,9698
841
843
  datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
@@ -931,7 +933,7 @@ datahub/sql_parsing/_sqlglot_patch.py,sha256=dpTiGj4Jxzvbi6HkYiOTbZ81CTFwEEo6JgY
931
933
  datahub/sql_parsing/datajob.py,sha256=1X8KpEk-y3_8xJuA_Po27EHZgOcxK9QADI6Om9gSGn0,1751
932
934
  datahub/sql_parsing/query_types.py,sha256=FKjDzszZzsrCfYfm7dgD6T_8865qxWl767fdGyHWBh4,2720
933
935
  datahub/sql_parsing/schema_resolver.py,sha256=ISuingLcQnOJZkNXBkc73uPwYUbbOtERAjgGhJajDiQ,10782
934
- datahub/sql_parsing/split_statements.py,sha256=6KUoIPG7H8Rja3lrPjSrSfhFfwW4oqgfoNQeTbbOWNg,8953
936
+ datahub/sql_parsing/split_statements.py,sha256=e1vVGJXqJbLPwjpFS6owROxVI6J0wwccyBV_9qxI0mc,9121
935
937
  datahub/sql_parsing/sql_parsing_aggregator.py,sha256=A3_0wSxBJSRowEaslptDpBoKO42XXx5UyTEK9PkekIs,70317
936
938
  datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
937
939
  datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
@@ -1041,8 +1043,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
1041
1043
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
1042
1044
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
1043
1045
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
1044
- acryl_datahub-1.0.0.1rc2.dist-info/METADATA,sha256=EQHrEsUieAKYQggwYr4Uhx4o7yhoXmtkPf9Y7XjdZ24,176849
1045
- acryl_datahub-1.0.0.1rc2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
1046
- acryl_datahub-1.0.0.1rc2.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
1047
- acryl_datahub-1.0.0.1rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1048
- acryl_datahub-1.0.0.1rc2.dist-info/RECORD,,
1046
+ acryl_datahub-1.0.0.1rc3.dist-info/METADATA,sha256=BTS0H-lGX3ONF9A0odwU_U8VbAUTrcd7K56DDTjG9iM,176849
1047
+ acryl_datahub-1.0.0.1rc3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
1048
+ acryl_datahub-1.0.0.1rc3.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
1049
+ acryl_datahub-1.0.0.1rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1050
+ acryl_datahub-1.0.0.1rc3.dist-info/RECORD,,
datahub/_version.py CHANGED
@@ -1,6 +1,6 @@
1
1
  # Published at https://pypi.org/project/acryl-datahub/.
2
2
  __package_name__ = "acryl-datahub"
3
- __version__ = "1.0.0.1rc2"
3
+ __version__ = "1.0.0.1rc3"
4
4
 
5
5
 
6
6
  def is_dev_mode() -> bool:
@@ -24,9 +24,9 @@ from typing import (
24
24
 
25
25
  import pydantic
26
26
  import requests
27
- from deprecated import deprecated
28
27
  from requests.adapters import HTTPAdapter, Retry
29
28
  from requests.exceptions import HTTPError, RequestException
29
+ from typing_extensions import deprecated
30
30
 
31
31
  from datahub._version import nice_version_name
32
32
  from datahub.cli import config_utils
@@ -597,7 +597,7 @@ class DataHubRestEmitter(Closeable, Emitter):
597
597
 
598
598
  return len(mcp_obj_chunks)
599
599
 
600
- @deprecated
600
+ @deprecated("Use emit with a datasetUsageStatistics aspect instead")
601
601
  def emit_usage(self, usageStats: UsageAggregation) -> None:
602
602
  url = f"{self._gms_server}/usageStats?action=batchIngest"
603
603
 
@@ -23,9 +23,9 @@ from typing import (
23
23
  )
24
24
 
25
25
  from avro.schema import RecordSchema
26
- from deprecated import deprecated
27
26
  from pydantic import BaseModel
28
27
  from requests.models import HTTPError
28
+ from typing_extensions import deprecated
29
29
 
30
30
  from datahub.cli import config_utils
31
31
  from datahub.configuration.common import ConfigModel, GraphError, OperationalError
@@ -330,7 +330,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
330
330
  f"Failed to find {aspect_type_name} in response {response_json}"
331
331
  )
332
332
 
333
- @deprecated(reason="Use get_aspect instead which makes aspect string name optional")
333
+ @deprecated("Use get_aspect instead which makes aspect string name optional")
334
334
  def get_aspect_v2(
335
335
  self,
336
336
  entity_urn: str,
@@ -355,7 +355,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
355
355
  def get_schema_metadata(self, entity_urn: str) -> Optional[SchemaMetadataClass]:
356
356
  return self.get_aspect(entity_urn=entity_urn, aspect_type=SchemaMetadataClass)
357
357
 
358
- @deprecated(reason="Use get_aspect directly.")
358
+ @deprecated("Use get_aspect directly.")
359
359
  def get_domain_properties(self, entity_urn: str) -> Optional[DomainPropertiesClass]:
360
360
  return self.get_aspect(entity_urn=entity_urn, aspect_type=DomainPropertiesClass)
361
361
 
@@ -376,7 +376,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
376
376
  def get_domain(self, entity_urn: str) -> Optional[DomainsClass]:
377
377
  return self.get_aspect(entity_urn=entity_urn, aspect_type=DomainsClass)
378
378
 
379
- @deprecated(reason="Use get_aspect directly.")
379
+ @deprecated("Use get_aspect directly.")
380
380
  def get_browse_path(self, entity_urn: str) -> Optional[BrowsePathsClass]:
381
381
  return self.get_aspect(entity_urn=entity_urn, aspect_type=BrowsePathsClass)
382
382
 
@@ -505,7 +505,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
505
505
  return response.json()
506
506
 
507
507
  @deprecated(
508
- reason="Use get_aspect for a single aspect or get_entity_semityped for a full entity."
508
+ "Use get_aspect for a single aspect or get_entity_semityped for a full entity."
509
509
  )
510
510
  def get_aspects_for_entity(
511
511
  self,
@@ -635,9 +635,6 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
635
635
  def _aspect_count_endpoint(self):
636
636
  return f"{self.config.server}/aspects?action=getCount"
637
637
 
638
- # def _session(self) -> Session:
639
- # return super()._session
640
-
641
638
  def get_domain_urn_by_name(self, domain_name: str) -> Optional[str]:
642
639
  """Retrieve a domain urn based on its name. Returns None if there is no match found"""
643
640
 
@@ -749,9 +746,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
749
746
 
750
747
  assert res["upsertConnection"]["urn"] == urn
751
748
 
752
- @deprecated(
753
- reason='Use get_urns_by_filter(entity_types=["container"], ...) instead'
754
- )
749
+ @deprecated('Use get_urns_by_filter(entity_types=["container"], ...) instead')
755
750
  def get_container_urns_by_filter(
756
751
  self,
757
752
  env: Optional[str] = None,
@@ -69,7 +69,7 @@ class FlowContainerSubTypes(StrEnum):
69
69
  class JobContainerSubTypes(StrEnum):
70
70
  NIFI_PROCESS_GROUP = "Process Group"
71
71
  MSSQL_JOBSTEP = "Job Step"
72
- MSSQL_STORED_PROCEDURE = "Stored Procedure"
72
+ STORED_PROCEDURE = "Stored Procedure"
73
73
 
74
74
 
75
75
  class BIAssetSubTypes(StrEnum):
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import os
2
3
  import time
3
4
  from dataclasses import dataclass
4
5
  from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union
@@ -115,6 +116,13 @@ class MLflowConfig(StatefulIngestionConfigBase, EnvConfigMixin):
115
116
  default=None, description="Mapping of source type to datahub platform"
116
117
  )
117
118
 
119
+ username: Optional[str] = Field(
120
+ default=None, description="Username for MLflow authentication"
121
+ )
122
+ password: Optional[str] = Field(
123
+ default=None, description="Password for MLflow authentication"
124
+ )
125
+
118
126
 
119
127
  @dataclass
120
128
  class MLflowRegisteredModelStageInfo:
@@ -161,7 +169,17 @@ class MLflowSource(StatefulIngestionSourceBase):
161
169
  self.ctx = ctx
162
170
  self.config = config
163
171
  self.report = StaleEntityRemovalSourceReport()
164
- self.client = MlflowClient(
172
+ self.client = self._configure_client()
173
+
174
+ def _configure_client(self) -> MlflowClient:
175
+ if bool(self.config.username) != bool(self.config.password):
176
+ raise ValueError("Both username and password must be set together")
177
+
178
+ if self.config.username and self.config.password:
179
+ os.environ["MLFLOW_TRACKING_USERNAME"] = self.config.username
180
+ os.environ["MLFLOW_TRACKING_PASSWORD"] = self.config.password
181
+
182
+ return MlflowClient(
165
183
  tracking_uri=self.config.tracking_uri,
166
184
  registry_uri=self.config.registry_uri,
167
185
  )
@@ -54,6 +54,7 @@ class SnowflakeObjectDomain(StrEnum):
54
54
  COLUMN = "column"
55
55
  ICEBERG_TABLE = "iceberg table"
56
56
  STREAM = "stream"
57
+ PROCEDURE = "procedure"
57
58
 
58
59
 
59
60
  GENERIC_PERMISSION_ERROR_KEY = "permission-error"
@@ -100,7 +100,15 @@ class SnowflakeFilterConfig(SQLFilterConfig):
100
100
 
101
101
  stream_pattern: AllowDenyPattern = Field(
102
102
  default=AllowDenyPattern.allow_all(),
103
- description="Regex patterns for streams to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'",
103
+ description="Regex patterns for streams to filter in ingestion. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'",
104
+ )
105
+
106
+ procedure_pattern: AllowDenyPattern = Field(
107
+ default=AllowDenyPattern.allow_all(),
108
+ description="Regex patterns for procedures to filter in ingestion. "
109
+ "Specify regex to match the entire procedure name in database.schema.procedure format. "
110
+ "e.g. to match all procedures starting with customer in Customer database and public schema,"
111
+ " use the regex 'Customer.public.customer.*'",
104
112
  )
105
113
 
106
114
  match_fully_qualified_names: bool = Field(
@@ -284,6 +292,11 @@ class SnowflakeV2Config(
284
292
  description="If enabled, streams will be ingested as separate entities from tables/views.",
285
293
  )
286
294
 
295
+ include_procedures: bool = Field(
296
+ default=True,
297
+ description="If enabled, procedures will be ingested as pipelines/tasks.",
298
+ )
299
+
287
300
  structured_property_pattern: AllowDenyPattern = Field(
288
301
  default=AllowDenyPattern.allow_all(),
289
302
  description=(
@@ -164,6 +164,23 @@ class SnowflakeQuery:
164
164
  and table_type in ('BASE TABLE', 'EXTERNAL TABLE')
165
165
  order by table_schema, table_name"""
166
166
 
167
+ @staticmethod
168
+ def procedures_for_database(db_name: Optional[str]) -> str:
169
+ db_clause = f'"{db_name}".' if db_name is not None else ""
170
+ return f"""
171
+ SELECT procedure_catalog AS "PROCEDURE_CATALOG",
172
+ procedure_schema AS "PROCEDURE_SCHEMA",
173
+ procedure_name AS "PROCEDURE_NAME",
174
+ procedure_language AS "PROCEDURE_LANGUAGE",
175
+ argument_signature AS "ARGUMENT_SIGNATURE",
176
+ data_type AS "PROCEDURE_RETURN_TYPE",
177
+ procedure_definition AS "PROCEDURE_DEFINITION",
178
+ created AS "CREATED",
179
+ last_altered AS "LAST_ALTERED",
180
+ comment AS "COMMENT"
181
+ FROM {db_clause}information_schema.procedures
182
+ order by procedure_schema, procedure_name"""
183
+
167
184
  @staticmethod
168
185
  def get_all_tags():
169
186
  return """
@@ -105,6 +105,7 @@ class SnowflakeV2Report(
105
105
  databases_scanned: int = 0
106
106
  tags_scanned: int = 0
107
107
  streams_scanned: int = 0
108
+ procedures_scanned: int = 0
108
109
 
109
110
  include_usage_stats: bool = False
110
111
  include_operational_stats: bool = False
@@ -163,6 +164,8 @@ class SnowflakeV2Report(
163
164
  self.tags_scanned += 1
164
165
  elif ent_type == "stream":
165
166
  self.streams_scanned += 1
167
+ elif ent_type == "procedure":
168
+ self.procedures_scanned += 1
166
169
  else:
167
170
  raise KeyError(f"Unknown entity {ent_type}.")
168
171
 
@@ -14,6 +14,7 @@ from datahub.ingestion.source.snowflake.snowflake_query import (
14
14
  SnowflakeQuery,
15
15
  )
16
16
  from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView
17
+ from datahub.ingestion.source.sql.stored_procedures.base import BaseProcedure
17
18
  from datahub.utilities.file_backed_collections import FileBackedDict
18
19
  from datahub.utilities.prefix_batch_builder import PrefixGroup, build_prefix_batches
19
20
  from datahub.utilities.serialized_lru_cache import serialized_lru_cache
@@ -714,3 +715,31 @@ class SnowflakeDataDictionary(SupportsAsObj):
714
715
  stream_pagination_marker = stream_name
715
716
 
716
717
  return streams
718
+
719
+ @serialized_lru_cache(maxsize=1)
720
+ def get_procedures_for_database(
721
+ self, db_name: str
722
+ ) -> Dict[str, List[BaseProcedure]]:
723
+ procedures: Dict[str, List[BaseProcedure]] = {}
724
+ cur = self.connection.query(
725
+ SnowflakeQuery.procedures_for_database(db_name),
726
+ )
727
+
728
+ for procedure in cur:
729
+ if procedure["PROCEDURE_SCHEMA"] not in procedures:
730
+ procedures[procedure["PROCEDURE_SCHEMA"]] = []
731
+
732
+ procedures[procedure["PROCEDURE_SCHEMA"]].append(
733
+ BaseProcedure(
734
+ name=procedure["PROCEDURE_NAME"],
735
+ language=procedure["PROCEDURE_LANGUAGE"],
736
+ argument_signature=procedure["ARGUMENT_SIGNATURE"],
737
+ return_type=procedure["PROCEDURE_RETURN_TYPE"],
738
+ procedure_definition=procedure["PROCEDURE_DEFINITION"],
739
+ created=procedure["CREATED"],
740
+ last_altered=procedure["LAST_ALTERED"],
741
+ comment=procedure["COMMENT"],
742
+ extra_properties=None,
743
+ )
744
+ )
745
+ return procedures