acryl-datahub 0.15.0.5rc8__py3-none-any.whl → 0.15.0.5rc9__py3-none-any.whl

{acryl_datahub-0.15.0.5rc8.dist-info → acryl_datahub-0.15.0.5rc9.dist-info}/RECORD

@@ -1,7 +1,7 @@
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=CilbE6n7KTTI5xCR6qR6cgnmqSiSFBe-Kp22mwO5MGQ,324
-datahub/entrypoints.py,sha256=56zAREkeLjycFIl0fubt9Haido463IUgl1QzPfcmAac,8344
+datahub/_version.py,sha256=KhEsNpGTq01OO4NFbvH7t1xq-yYSpEDm5CLJZkdZi6o,324
+datahub/entrypoints.py,sha256=2TYgHhs3sCxJlojIHjqfxzt3_ImPwPzq4vBtsUuMqu4,8885
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/_codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/_codegen/aspect.py,sha256=PJRa-Z4ouXHq3OkulfyWhwZn-fFUBDK_UPvmqaWdbWk,1063
@@ -134,7 +134,7 @@ datahub/ingestion/api/committable.py,sha256=4S6GuBzvX2vb1A8P506NbspOKfZ1621sBG8t
 datahub/ingestion/api/common.py,sha256=nJVL8YdvokYFajOjmVpSNlLbZJ5iVOFS4KJDlGtJ_jc,2735
 datahub/ingestion/api/decorators.py,sha256=b9bxHXlqCLDgqrVdPU6WNQg1koZcK62AkZ9vNwvWeK4,4029
 datahub/ingestion/api/global_context.py,sha256=OdSJg4a_RKE52nu8MSiEkK2UqRRDhDTyOleHEAzPKho,575
-datahub/ingestion/api/incremental_lineage_helper.py,sha256=JTmJvXzzwI04oTUTIeTKKscT_hjnr8nW34NFWJvCXDc,5871
+datahub/ingestion/api/incremental_lineage_helper.py,sha256=7a6FTJ_uz4EEJS1vPtbYB2KvNlcZB3py28_FKxmRiSk,5993
 datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPsCtRa7ffDGPA1w1hgPUjeenZBU,2514
 datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
 datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
@@ -163,8 +163,8 @@ datahub/ingestion/fs/http_fs.py,sha256=NBIKp4vl7mW0YfVfkfpO3R6DBGqSC7f6EE_da0yz2
 datahub/ingestion/fs/local_fs.py,sha256=oWf-PZsl5sI-9eHWGeKlfKYagbQaSZ9fGfNbxcFji14,885
 datahub/ingestion/fs/s3_fs.py,sha256=kGq4lWjTLCfPNinYfEiasADFyYb1PToD_8HXSisjdRY,3199
 datahub/ingestion/glossary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/glossary/classification_mixin.py,sha256=pkb0Rv2SQH7VwAV5DPLoJLJwkDwTjIhOhg4mbXiz9CI,13332
-datahub/ingestion/glossary/classifier.py,sha256=zp8Fe3he80H5Zz1EwymKjThUPkTpw6PgEJQvlmqrJmQ,3006
+datahub/ingestion/glossary/classification_mixin.py,sha256=3vMyHmo-nL1G8OijDQk1XSBxkmXv2ziyUSPeUIUSLSI,13789
+datahub/ingestion/glossary/classifier.py,sha256=daLxnVv_JlfB_jBOxH5LrU_xQRndrsGot6z9Cir5Vuc,2981
 datahub/ingestion/glossary/classifier_registry.py,sha256=yFOYLQhDgCLqXYMG3L1BquXafeLcZDcmp8meyw6k9ts,307
 datahub/ingestion/glossary/datahub_classifier.py,sha256=O7wm6gQT1Jf2QSKdWjJQbS5oSzJwplXzfza26Gdq5Mg,7555
 datahub/ingestion/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -201,7 +201,7 @@ datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suab
 datahub/ingestion/source/ldap.py,sha256=Vnzg8tpwBYeyM-KBVVsUJvGZGBMJiCJ_i_FhxaFRQ9A,18627
 datahub/ingestion/source/metabase.py,sha256=m9Gfhrs8F1z23ci8CIxdE5cW--25stgxg_IQTKwkFrk,31532
 datahub/ingestion/source/mlflow.py,sha256=pmIkmsfidi7dOGdQ61rab7m8AnKZhIRE2IA9in9HGFU,12144
-datahub/ingestion/source/mode.py,sha256=XeWL2yX2aJ_jC15jgseetBNswhtPOZnySlRspy9LZmY,63499
+datahub/ingestion/source/mode.py,sha256=HVxhzMIY4HjkAG_T6y00Po2B9XwjALP6i5XQThuyYM4,63488
 datahub/ingestion/source/mongodb.py,sha256=Hucd3rfxwRcc_rNOJbpSPmSZdKqN6Fi9L7KcUZ80YKM,21104
 datahub/ingestion/source/nifi.py,sha256=BszXfFonfHB63Zt85lHDh4W_V-gIJKtxS6q3cdPDc4U,56021
 datahub/ingestion/source/openapi.py,sha256=MGsRLseZompW10UVMN_tU1GZgqPgTAM4lnqCJ8eVRoY,17386
@@ -324,7 +324,7 @@ datahub/ingestion/source/iceberg/iceberg.py,sha256=2j-MKCa0o6m1btlgYssYTEAjcD6Zp
 datahub/ingestion/source/iceberg/iceberg_common.py,sha256=LEZaJleL5KJt1u_pLRUkeCqPEsthzH7tG8FgBwd9MC8,10218
 datahub/ingestion/source/iceberg/iceberg_profiler.py,sha256=CkBB5fryMVoqqCM6eLSIeb4yP85ABHONNRm0QqZKrnw,9977
 datahub/ingestion/source/identity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/identity/azure_ad.py,sha256=OEkd7X7w2H4mrMo3xXUfzEQlALoogjR2QabZm8CrAW0,28811
+datahub/ingestion/source/identity/azure_ad.py,sha256=9Hrvm4CSfc02yjnPUsCYSY4Qw9fXPnDFWLexab0mcpc,28559
 datahub/ingestion/source/identity/okta.py,sha256=LMbW5N1j9kMjcvGnmcff8LpIDPwmscmPPOTZD88KZZg,30758
 datahub/ingestion/source/kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/kafka/kafka.py,sha256=ZK2NQi5GRobruwn58LR6JMKsnZl269YZzgYAMyI1Y3s,26504
@@ -435,7 +435,7 @@ datahub/ingestion/source/snowflake/constants.py,sha256=22n-0r04nuy-ImxWFFpmbrt_G
 datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
 datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
 datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
-datahub/ingestion/source/snowflake/snowflake_config.py,sha256=V07ELzDFcQMIBX89ojKoJDxEOJzq_eCKsMehhmuCfYI,19231
+datahub/ingestion/source/snowflake/snowflake_config.py,sha256=Y3LoqBavhc3Cm0nyAr3fnd_-i4gReDfaAuUdp7EgwPQ,19603
 datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=e9dCARIQtGB8G1cSMRLorCbNLcPUD2g9gBL-LLLKjFE,17793
 datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
 datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=FBmiONx4EGHWV8RNJT6zHZyntKinPFFyd2oKbTUIbhE,21319
@@ -450,7 +450,7 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYh
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=1eLYTcgmfzDs9xktMTTE74L5SeNP48Qg3uLr9y-Ez3Y,8733
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
 datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=xq58c47zmaQPkTVqjKW25iViX8VJuHdQDTFY4jxzZ2o,12778
-datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=vyTqC_C5Bf0AMRVyoxUfl1CdlgeQouX20msP2FsMqnk,33439
+datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=NidPSzXh2UajcvgeDoTmk31UW1dAeQBCCFjumZajzcI,33524
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=Uh9wGLOqAkcphffxOPIQNyXvjeRm74XIpaLb4rjqMjM,24045
 datahub/ingestion/source/sql/clickhouse.py,sha256=uSRy-HKAiGFTHVLoVtGoh23X0O1lwyYUaK8BaWkYhps,25555
@@ -573,7 +573,7 @@ datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1
 datahub/lite/lite_util.py,sha256=pgBpT3vTO1YCQ2njZRNyicSkHYeEmQCt41BaXU8WvMo,4503
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
 datahub/metadata/_schema_classes.py,sha256=xDKwU0EtWDoHPgetCMWFYK80SRbZWONFE4HXApDZGo8,985155
-datahub/metadata/schema.avsc,sha256=ISpdTivunuDG6EebIw8N0oWkUrsc-h9N4q1XcfdDhxk,646307
+datahub/metadata/schema.avsc,sha256=i4ukol5SJl7lyGsw0cXfOirqVnQfAVzQx5tUDH-n-EU,646297
 datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -678,7 +678,7 @@ datahub/metadata/schemas/CorpUserKey.avsc,sha256=-Spvvcss0sJoADygdChWa99rYiMMRwE
 datahub/metadata/schemas/CorpUserSettings.avsc,sha256=fS2HUD0L9_rsPyqo0DRxibzPi8_IkkWTY6Zuqob1sPg,2097
 datahub/metadata/schemas/CorpUserStatus.avsc,sha256=yqojAXEQ9CjRhY58RPyTUxzmFbHSANGGaMMbqiYZZIE,2538
 datahub/metadata/schemas/Cost.avsc,sha256=o4kYZSss2uEwJ6gCA9fhBUoyD5xUqcSxz78vkIXXzGQ,1494
-datahub/metadata/schemas/DashboardInfo.avsc,sha256=kuRyOSQWRvV9ydhozTWKWrNfbD8ZNeWawGGR4xSHhaI,12917
+datahub/metadata/schemas/DashboardInfo.avsc,sha256=li2lSV6R4V-nz6foOi-NYxt_8ShHWfoKRw6M2BG5530,12907
 datahub/metadata/schemas/DashboardKey.avsc,sha256=yKlusgebWTvZhVeGgRNLZW6Qu6Fg_K0e2EbV8zr3jvA,1360
 datahub/metadata/schemas/DashboardUsageStatistics.avsc,sha256=pUAKqs49Wy5pAL92g_6QcFtJeoYeMWRGiHWS68IJN2A,7693
 datahub/metadata/schemas/DataContractKey.avsc,sha256=Oceu7P26--E0812IFrX3RiEY0Ktam869iiYN30zBudc,481
@@ -813,7 +813,7 @@ datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKj
 datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=mX4CQcoN3FC_VQDBCkhlmJk4pfQKDrSeuqqCTTXTmq8,1092
 datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
 datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
-datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=pbt_zFR9fPRm5ek1cuYa1eMCNs3aD7F6jOruQaee2NY,373224
+datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=73m6GXjSHTILKxczi7IxN8Dm0iBFTps0QKph0AkaRag,373214
 datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=mpdodpx25E6M1Gq_7slEcPAm-1Es5xPsoqV60HgO7zg,12167
 datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=EMfQrYsuHf1p6UvBjoLtfdTHGe-vGNJaCFEHz8hdKU0,9698
 datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
@@ -875,7 +875,7 @@ datahub/secret/secret_common.py,sha256=g4anQtYPm7cI6kEJUZHjpBqeCyiUKIim2rJQByaeO
 datahub/secret/secret_store.py,sha256=2VP_Vd336Cy7C-2kwp4rx8MAqtYgtwv8XyzzNTXE5x8,1124
 datahub/specific/__init__.py,sha256=r5RYM5mDnskLzin3vc87HV-9GSz3P6uQw8AlsN14LaI,88
 datahub/specific/chart.py,sha256=NPdzDHcZkPodthOn9c8QF_aDEo2y4lCJ4t1sI556uZ0,6684
-datahub/specific/dashboard.py,sha256=D8CnOSScQ0-UICFjQnQOtqL-SlNSxhSuub4vZ3BpcuI,10017
+datahub/specific/dashboard.py,sha256=3AsXZ1Cp03uaTHsOmJqEiXzJjZUBgDbX-zmgwMw908o,11514
 datahub/specific/datajob.py,sha256=yZXic3CuCGxg-ewnaHphoH9Jjpe-P09XbN7T-rrvkyE,13711
 datahub/specific/dataproduct.py,sha256=SrBNYACDVatuXA64GCHA0Igaes24ajJqTcXmDTT1FPA,2877
 datahub/specific/dataset.py,sha256=je9j3rVzpSiXoOe0UmfD7mc5vCpLAAO74Z8q1SvwPX0,9725
@@ -912,7 +912,7 @@ datahub/testing/docker_utils.py,sha256=g169iy_jNR_mg0p8X31cChZqjOryutAIHUYLq3xqu
 datahub/testing/doctest.py,sha256=1_8WEhHZ2eRQtw8vsXKzr9L5zzvs0Tcr6q4mnkyyvtw,295
 datahub/testing/mcp_diff.py,sha256=Dxde5uZHqZf1EjOkHm405OHY5PPJp03agZJM9SyR4yE,10717
 datahub/upgrade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/upgrade/upgrade.py,sha256=Qy0pHqxpIPbD16mUcyk6Wkq9vbvnUPvtI5d9fovrY3Q,16338
+datahub/upgrade/upgrade.py,sha256=iDjIDY2YBl2XlKLvb5EMMdYOZ6KraeItgiu9Y4wIM1Q,16666
 datahub/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/utilities/_custom_package_loader.py,sha256=9kgPE7Y77E-hNee8l4sKtVby-btUNum3dBfDixMzcVA,2059
 datahub/utilities/_markupsafe_compat.py,sha256=QX7c9KiHs56ASl7bJlgR4FAf3CGiY94zIr0h6Ak15To,444
@@ -1000,9 +1000,9 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-0.15.0.5rc8.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
-acryl_datahub-0.15.0.5rc8.dist-info/METADATA,sha256=dv5hJaDf3GORg52Q4TCqkatjmEdPrpFbIgIE22kxvAY,175284
-acryl_datahub-0.15.0.5rc8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-acryl_datahub-0.15.0.5rc8.dist-info/entry_points.txt,sha256=U1e5ZwqPX1OaIbvGrwvozcdB8SbzFYXQM7plpdLKKeo,9592
-acryl_datahub-0.15.0.5rc8.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-0.15.0.5rc8.dist-info/RECORD,,
+acryl_datahub-0.15.0.5rc9.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-0.15.0.5rc9.dist-info/METADATA,sha256=VuKgWOBCJTCnPCsvt0eB4LoZYj-ig56pUIgrs86xB7w,175375
+acryl_datahub-0.15.0.5rc9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+acryl_datahub-0.15.0.5rc9.dist-info/entry_points.txt,sha256=U1e5ZwqPX1OaIbvGrwvozcdB8SbzFYXQM7plpdLKKeo,9592
+acryl_datahub-0.15.0.5rc9.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-0.15.0.5rc9.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "0.15.0.5rc8"
+__version__ = "0.15.0.5rc9"
 
 
 def is_dev_mode() -> bool:

datahub/entrypoints.py

@@ -1,4 +1,5 @@
 import logging
+import multiprocessing
 import os
 import platform
 import sys
@@ -217,6 +218,14 @@ except ImportError as e:
 
 
 def main(**kwargs):
+    # We use threads in a variety of places within our CLI. The multiprocessing
+    # "fork" start method is not safe to use with threads.
+    # MacOS and Windows already default to "spawn", and Linux will as well starting in Python 3.14.
+    # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+    # Eventually it may make sense to use "forkserver" as the default where available,
+    # but we can revisit that in the future.
+    multiprocessing.set_start_method("spawn", force=True)
+
     # This wrapper prevents click from suppressing errors.
     try:
         sys.exit(datahub(standalone_mode=False, **kwargs))

datahub/ingestion/api/incremental_lineage_helper.py

@@ -102,6 +102,10 @@ def convert_dashboard_info_to_patch(
     if aspect.datasets:
         patch_builder.add_datasets(aspect.datasets)
 
+    if aspect.dashboards:
+        for dashboard in aspect.dashboards:
+            patch_builder.add_dashboard(dashboard)
+
     if aspect.access:
         patch_builder.set_access(aspect.access)
 

datahub/ingestion/glossary/classification_mixin.py

@@ -1,5 +1,6 @@
 import concurrent.futures
 import logging
+import multiprocessing
 from dataclasses import dataclass, field
 from functools import partial
 from math import ceil
@@ -182,6 +183,11 @@ class ClassificationHandler:
 
         with concurrent.futures.ProcessPoolExecutor(
             max_workers=self.config.classification.max_workers,
+            # The fork start method, which is the default on Linux for Python < 3.14, is not
+            # safe when the main process uses threads. The default start method on windows/macOS is
+            # already spawn, and will be changed to spawn for Linux in Python 3.14.
+            # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+            mp_context=multiprocessing.get_context("spawn"),
         ) as executor:
             column_info_proposal_futures = [
                 executor.submit(

datahub/ingestion/glossary/classifier.py

@@ -1,3 +1,4 @@
+import os
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional
@@ -37,8 +38,8 @@ class ClassificationConfig(ConfigModel):
     )
 
     max_workers: int = Field(
-        default=1,
-        description="Number of worker processes to use for classification. Note that any number above 1 might lead to a deadlock. Set to 1 to disable.",
+        default=(os.cpu_count() or 4),
+        description="Number of worker processes to use for classification. Set to 1 to disable.",
     )
 
     table_pattern: AllowDenyPattern = Field(

datahub/ingestion/source/identity/azure_ad.py

@@ -13,6 +13,7 @@ from requests.adapters import HTTPAdapter, Retry
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.configuration.source_common import DatasetSourceConfigMixin
+from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.emitter.mce_builder import make_group_urn, make_user_urn
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
@@ -51,6 +52,7 @@ from datahub.metadata.schema_classes import (
     OriginTypeClass,
     StatusClass,
 )
+from datahub.utilities.lossy_collections import LossyList
 
 logger = logging.getLogger(__name__)
 
@@ -132,11 +134,7 @@ class AzureADConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
         description="regex patterns for groups to include in ingestion.",
     )
 
-    # If enabled, report will contain names of filtered users and groups.
-    filtered_tracking: bool = Field(
-        default=True,
-        description="If enabled, report will contain names of filtered users and groups.",
-    )
+    _remove_filtered_tracking = pydantic_removed_field("filtered_tracking")
 
     # Optional: Whether to mask sensitive information from workunit ID's. On by default.
     mask_group_id: bool = Field(
@@ -156,14 +154,10 @@ class AzureADConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin):
 
 @dataclass
 class AzureADSourceReport(StaleEntityRemovalSourceReport):
-    filtered: List[str] = field(default_factory=list)
-    filtered_tracking: bool = field(default=True, repr=False)
-    filtered_count: int = field(default=0)
+    filtered: LossyList[str] = field(default_factory=LossyList)
 
     def report_filtered(self, name: str) -> None:
-        self.filtered_count += 1
-        if self.filtered_tracking:
-            self.filtered.append(name)
+        self.filtered.append(name)
 
 
 # Source that extracts Azure AD users, groups and group memberships using Microsoft Graph REST API
@@ -266,9 +260,7 @@ class AzureADSource(StatefulIngestionSourceBase):
     def __init__(self, config: AzureADConfig, ctx: PipelineContext):
         super().__init__(config, ctx)
         self.config = config
-        self.report = AzureADSourceReport(
-            filtered_tracking=self.config.filtered_tracking
-        )
+        self.report = AzureADSourceReport()
         session = requests.Session()
         retries = Retry(
             total=5, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]

datahub/ingestion/source/mode.py

@@ -24,6 +24,7 @@ from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponenti
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import AllowDenyPattern, ConfigModel
 from datahub.configuration.source_common import DatasetLineageProviderConfigBase
+from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import (
     ContainerKey,
@@ -155,10 +156,7 @@ class ModeConfig(StatefulIngestionConfigBase, DatasetLineageProviderConfigBase):
     workspace: str = Field(
         description="The Mode workspace name. Find it in Settings > Workspace > Details."
     )
-    default_schema: str = Field(
-        default="public",
-        description="Default schema to use when schema is not provided in an SQL query",
-    )
+    _default_schema = pydantic_removed_field("default_schema")
 
     space_pattern: AllowDenyPattern = Field(
         default=AllowDenyPattern(

datahub/ingestion/source/snowflake/snowflake_config.py

@@ -308,6 +308,13 @@ class SnowflakeV2Config(
         " assertions CLI in snowflake",
     )
 
+    pushdown_deny_usernames: List[str] = Field(
+        default=[],
+        description="List of snowflake usernames which will not be considered for lineage/usage/queries extraction. "
+        "This is primarily useful for improving performance by filtering out users with extremely high query volumes. "
+        "Only applicable if `use_queries_v2` is enabled.",
+    )
+
     @validator("convert_urns_to_lowercase")
     def validate_convert_urns_to_lowercase(cls, v):
         if not v:

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -567,6 +567,7 @@ class SnowflakeV2Source(
                         include_queries=self.config.include_queries,
                         include_query_usage_statistics=self.config.include_query_usage_statistics,
                         user_email_pattern=self.config.user_email_pattern,
+                        pushdown_deny_usernames=self.config.pushdown_deny_usernames,
                     ),
                     structured_report=self.report,
                     filters=self.filters,

datahub/metadata/schema.avsc

@@ -4730,16 +4730,16 @@
       {
         "Relationship": {
           "/*/destinationUrn": {
-            "createdActor": "datasetEdges/*/created/actor",
-            "createdOn": "datasetEdges/*/created/time",
+            "createdActor": "dashboards/*/created/actor",
+            "createdOn": "dashboards/*/created/time",
             "entityTypes": [
               "dashboard"
             ],
             "isLineage": true,
             "name": "DashboardContainsDashboard",
-            "properties": "datasetEdges/*/properties",
-            "updatedActor": "datasetEdges/*/lastModified/actor",
-            "updatedOn": "datasetEdges/*/lastModified/time"
+            "properties": "dashboards/*/properties",
+            "updatedActor": "dashboards/*/lastModified/actor",
+            "updatedOn": "dashboards/*/lastModified/time"
           }
         },
         "type": {

datahub/metadata/schemas/DashboardInfo.avsc

@@ -258,16 +258,16 @@
     {
       "Relationship": {
         "/*/destinationUrn": {
-          "createdActor": "datasetEdges/*/created/actor",
-          "createdOn": "datasetEdges/*/created/time",
+          "createdActor": "dashboards/*/created/actor",
+          "createdOn": "dashboards/*/created/time",
           "entityTypes": [
             "dashboard"
           ],
           "isLineage": true,
           "name": "DashboardContainsDashboard",
-          "properties": "datasetEdges/*/properties",
-          "updatedActor": "datasetEdges/*/lastModified/actor",
-          "updatedOn": "datasetEdges/*/lastModified/time"
+          "properties": "dashboards/*/properties",
+          "updatedActor": "dashboards/*/lastModified/actor",
+          "updatedOn": "dashboards/*/lastModified/time"
         }
       },
       "type": {

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -2049,16 +2049,16 @@
                       {
                         "Relationship": {
                           "/*/destinationUrn": {
-                            "createdActor": "datasetEdges/*/created/actor",
-                            "createdOn": "datasetEdges/*/created/time",
+                            "createdActor": "dashboards/*/created/actor",
+                            "createdOn": "dashboards/*/created/time",
                             "entityTypes": [
                               "dashboard"
                             ],
                             "isLineage": true,
                             "name": "DashboardContainsDashboard",
-                            "properties": "datasetEdges/*/properties",
-                            "updatedActor": "datasetEdges/*/lastModified/actor",
-                            "updatedOn": "datasetEdges/*/lastModified/time"
+                            "properties": "dashboards/*/properties",
+                            "updatedActor": "dashboards/*/lastModified/actor",
+                            "updatedOn": "dashboards/*/lastModified/time"
                           }
                         },
                         "type": {

datahub/specific/dashboard.py

@@ -161,7 +161,7 @@ class DashboardPatchBuilder(
                 lastModified=self._mint_auditstamp(),
             )
 
-        self._ensure_urn_type("dataset", [chart_edge], "add_chart_edge")
+        self._ensure_urn_type("chart", [chart_edge], "add_chart_edge")
         self._add_patch(
             DashboardInfo.ASPECT_NAME,
             "add",
@@ -271,6 +271,48 @@ class DashboardPatchBuilder(
 
         return self
 
+    def add_dashboard(
+        self, dashboard: Union[Edge, Urn, str]
+    ) -> "DashboardPatchBuilder":
+        """
+        Adds an dashboard to the DashboardPatchBuilder.
+
+        Args:
+            dashboard: The dashboard, which can be an Edge object, Urn object, or a string.
+
+        Returns:
+            The DashboardPatchBuilder instance.
+
+        Raises:
+            ValueError: If the dashboard is not a Dashboard urn.
+
+        Notes:
+            If `dashboard` is an Edge object, it is used directly. If `dashboard` is a Urn object or string,
+            it is converted to an Edge object and added with default audit stamps.
+        """
+        if isinstance(dashboard, Edge):
+            dashboard_urn: str = dashboard.destinationUrn
+            dashboard_edge: Edge = dashboard
+        elif isinstance(dashboard, (Urn, str)):
+            dashboard_urn = str(dashboard)
+            if not dashboard_urn.startswith("urn:li:dashboard:"):
+                raise ValueError(f"Input {dashboard} is not a Dashboard urn")
+
+            dashboard_edge = Edge(
+                destinationUrn=dashboard_urn,
+                created=self._mint_auditstamp(),
+                lastModified=self._mint_auditstamp(),
+            )
+
+        self._ensure_urn_type("dashboard", [dashboard_edge], "add_dashboard")
+        self._add_patch(
+            DashboardInfo.ASPECT_NAME,
+            "add",
+            path=("dashboards", dashboard_urn),
+            value=dashboard_edge,
+        )
+        return self
+
     def set_dashboard_url(
         self, dashboard_url: Optional[str]
     ) -> "DashboardPatchBuilder":

datahub/upgrade/upgrade.py

@@ -55,11 +55,19 @@ async def get_client_version_stats():
         async with session.get(pypi_url) as resp:
             response_json = await resp.json()
             try:
-                releases = response_json.get("releases", [])
-                sorted_releases = sorted(releases.keys(), key=lambda x: Version(x))
-                latest_cli_release_string = [
-                    x for x in sorted_releases if "rc" not in x
-                ][-1]
+                releases = response_json.get("releases", {})
+                filtered_releases = {
+                    version: release_files
+                    for version, release_files in releases.items()
+                    if not all(
+                        release_file.get("yanked") for release_file in release_files
+                    )
+                    and "rc" not in version
+                }
+                sorted_releases = sorted(
+                    filtered_releases.keys(), key=lambda x: Version(x)
+                )
+                latest_cli_release_string = sorted_releases[-1]
                 latest_cli_release = Version(latest_cli_release_string)
                 current_version_info = releases.get(current_version_string)
                 current_version_date = None