acryl-datahub 1.1.0.5rc11__py3-none-any.whl → 1.1.0.5rc13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (29) hide show
  1. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/METADATA +2593 -2593
  2. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/RECORD +29 -27
  3. datahub/_version.py +1 -1
  4. datahub/cli/quickstart_versioning.py +4 -6
  5. datahub/ingestion/api/source.py +2 -1
  6. datahub/ingestion/source/abs/source.py +1 -1
  7. datahub/ingestion/source/azure/azure_common.py +2 -2
  8. datahub/ingestion/source/bigquery_v2/bigquery_config.py +1 -1
  9. datahub/ingestion/source/data_lake_common/object_store.py +75 -27
  10. datahub/ingestion/source/dbt/dbt_cloud.py +7 -2
  11. datahub/ingestion/source/dbt/dbt_common.py +2 -0
  12. datahub/ingestion/source/mock_data/datahub_mock_data.py +55 -13
  13. datahub/ingestion/source/snowflake/snowflake_config.py +15 -1
  14. datahub/ingestion/source/snowflake/snowflake_queries.py +57 -19
  15. datahub/ingestion/source/snowflake/snowflake_v2.py +2 -0
  16. datahub/ingestion/source/sql/athena.py +10 -0
  17. datahub/ingestion/source/sql/oracle.py +1 -1
  18. datahub/ingestion/source/tableau/tableau_common.py +4 -2
  19. datahub/metadata/_internal_schema_classes.py +35 -2
  20. datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
  21. datahub/metadata/schema.avsc +38 -11
  22. datahub/metadata/schemas/DatasetKey.avsc +2 -1
  23. datahub/metadata/schemas/LogicalParent.avsc +140 -0
  24. datahub/metadata/schemas/QuerySubjects.avsc +1 -12
  25. datahub/metadata/schemas/SchemaFieldKey.avsc +2 -1
  26. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/WHEEL +0 -0
  27. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/entry_points.txt +0 -0
  28. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/licenses/LICENSE +0 -0
  29. {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
- acryl_datahub-1.1.0.5rc11.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
1
+ acryl_datahub-1.1.0.5rc13.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
2
2
  datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
3
3
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
4
- datahub/_version.py,sha256=pW5LCAauFcN_5fwryjQJdB81K2HvRSufgySvocDiDdA,324
4
+ datahub/_version.py,sha256=AWm-T7Sf9DTy4rZWQzbDvsJbdT1nZJ1ux45nnicW6tI,324
5
5
  datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
6
6
  datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
7
7
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -84,7 +84,7 @@ datahub/cli/lite_cli.py,sha256=XKMejSuYUToKBvgN3YmmnxjRcaG5WPw23gJuQK8pgRc,13099
84
84
  datahub/cli/migrate.py,sha256=XWW_fd4EUQGAUrKh9oKmtNYmfhcvbqWZJj3TQNgD8Dk,18012
85
85
  datahub/cli/migration_utils.py,sha256=snkkT8ixo3Il4bvc7qN_QBOP1fz5UsO0usrUl2zJ68w,9478
86
86
  datahub/cli/put_cli.py,sha256=ehqsxEPVVK-0xu7nVfBl_GxfsDbkgjIjkF8mfGfcIpc,3827
87
- datahub/cli/quickstart_versioning.py,sha256=quLCjWBORu7LCkJ3_cquiSOguu0SRxl3R9hLqOekzYo,7433
87
+ datahub/cli/quickstart_versioning.py,sha256=Ec7SehEImRQW4I_8O89cTG1MvZtnnd8EJx9X8NaauoY,7302
88
88
  datahub/cli/state_cli.py,sha256=RDXYCb17sqjW49nNdkoGAqNPRRHOKw5lcgcs5QvHeOM,1142
89
89
  datahub/cli/telemetry.py,sha256=xw3SiAn2je48Qv4kXPYN5EPVKHWEWZc2LGoF7UzGs8U,489
90
90
  datahub/cli/timeline_cli.py,sha256=Wc_Mo7khFFHXgRSl2FgtvB8m0QhuCEhkpaFCIOPpHJE,7584
@@ -151,7 +151,7 @@ datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX
151
151
  datahub/ingestion/api/report.py,sha256=_bP7JPHZyftWnCJwFs6YId21mQ0-2rd406H91lLHW98,16942
152
152
  datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
153
153
  datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
154
- datahub/ingestion/api/source.py,sha256=WcpmuWRua87_UbaSY8GtkLkC0Gql0pM_i5FJN5ALh88,19669
154
+ datahub/ingestion/api/source.py,sha256=1vZu4oWLS1D8XMV55UjxOkOnMSQjnx0NMwVFFDS065Y,19694
155
155
  datahub/ingestion/api/source_helpers.py,sha256=XT9y5HgfVeF52jrX39vlLn1SdXpLVyT2Su8oGNsddYo,21148
156
156
  datahub/ingestion/api/transform.py,sha256=X0GpjMJzYkLuZx8MTWxH50cWGm9rGsnn3k188mmC8J8,582
157
157
  datahub/ingestion/api/workunit.py,sha256=e8n8RfSjHZZm2R4ShNH0UuMtUkMjyqqM2j2t7oL74lo,6327
@@ -234,7 +234,7 @@ datahub/ingestion/source/abs/config.py,sha256=mBQe0JTaP-Rcv4HnMUUySoYbSr4r3jDEMi
234
234
  datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=Rkf64evufyVGPiE4VK8QAjzBiJFu85tOGMmJ0lJZ2Og,3600
235
235
  datahub/ingestion/source/abs/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
236
236
  datahub/ingestion/source/abs/report.py,sha256=CkRjsNn0Pab-ZPllxz3IUJI_r3x0T6urJePa_hJKi5U,586
237
- datahub/ingestion/source/abs/source.py,sha256=cuMezUzr-Smp5tok2ceYor5I5jp52NDMjfeN8kfIbvg,24816
237
+ datahub/ingestion/source/abs/source.py,sha256=FvncxJG2ZjQ4Q5JeYo4zVYuqEBsfvHOP4ZSjy_CKG90,24826
238
238
  datahub/ingestion/source/apply/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
239
239
  datahub/ingestion/source/apply/datahub_apply.py,sha256=xTD-Iq3UHhxcz61RwNuI2kJjRrnQEfZFSgvS1X6loV4,7703
240
240
  datahub/ingestion/source/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -254,12 +254,12 @@ datahub/ingestion/source/aws/sagemaker_processors/models.py,sha256=N9FtnOznwwToc
254
254
  datahub/ingestion/source/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
255
255
  datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0ch1oF2SJSYDZ1JMB_Onso,7605
256
256
  datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
257
- datahub/ingestion/source/azure/azure_common.py,sha256=Zl0pPuE6L3QcM5B1P0LsPthZmD0h7fUUS0kg2okl6IY,4053
257
+ datahub/ingestion/source/azure/azure_common.py,sha256=jr9i0m__TuGJpEQ5BwHrQw1M9s8j_OEn-iZ8DPX6cuk,4043
258
258
  datahub/ingestion/source/bigquery_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
259
259
  datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=u4-LLt6ZDe3hKqLWqEByYpc0z-UcEZf85uok9qNEFko,15321
260
260
  datahub/ingestion/source/bigquery_v2/bigquery_audit.py,sha256=kEwWhq3ch6WT4q4hcX8-fvQh28KgrNfspFwIytO3vQA,25103
261
261
  datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py,sha256=LuGJ6LgPViLIfDQfylxlQ3CA7fZYM5MDt8M-7sfzm84,5096
262
- datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=bG1soRawCLhJw_15L2fJmFfj1kntTthV6ng4LZOnwko,21916
262
+ datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=DYfZQIwcB9kzf0Kv_m5sUhdBYtrl4LTriZZ7n7mK4-w,21915
263
263
  datahub/ingestion/source/bigquery_v2/bigquery_connection.py,sha256=sq9Nk11W5cMPs4X9ST6OBb3UnHkSeyBHtB_bKU1lOj8,2799
264
264
  datahub/ingestion/source/bigquery_v2/bigquery_data_reader.py,sha256=DeT3v_Z82__8En0FcZ0kavBAWQoRvSZ5Rppm9eeDAb8,2393
265
265
  datahub/ingestion/source/bigquery_v2/bigquery_helper.py,sha256=QER3gY8e_k1_eNVj7cBso7ZzrWl_vO5PYSa6CpvqNx8,1554
@@ -289,7 +289,7 @@ datahub/ingestion/source/common/subtypes.py,sha256=x8dv9SN23Y8t5oq7SUuQWUPAH8KM3
289
289
  datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
290
290
  datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
291
291
  datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGDTUOol9Oi3jZs8Pb04PIsHdC2WPP0dkg,6576
292
- datahub/ingestion/source/data_lake_common/object_store.py,sha256=NkbCXq56sdGkyBAxcrpSVE6GySLrh9E2jYZU6lYPupM,21644
292
+ datahub/ingestion/source/data_lake_common/object_store.py,sha256=i9Hgb8Ww23QD_jEjzj_2qxA8Nr56krnZfo1qyOWmH9M,23608
293
293
  datahub/ingestion/source/data_lake_common/path_spec.py,sha256=ekJAr4-PE2RhzQnmKb2xcSs_YncC1Dz95-UrXI67Vos,23584
294
294
  datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
295
295
  datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
@@ -300,8 +300,8 @@ datahub/ingestion/source/datahub/datahub_source.py,sha256=LsfDntDtqt_HMYCWwYctFK
300
300
  datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vBCU0XxGcZR6Xxs,940
301
301
  datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
302
302
  datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
303
- datahub/ingestion/source/dbt/dbt_cloud.py,sha256=435blM28-RIE7ZfQReh69zBXppv8tjbku10oFcyJY8Q,17891
304
- datahub/ingestion/source/dbt/dbt_common.py,sha256=5iqlHbH1eaRcysxvFmDfgNrfVpygnmmeHbLq_sLg2P4,82190
303
+ datahub/ingestion/source/dbt/dbt_cloud.py,sha256=4gWOFSX0YU8EAJgO4J47NBE4QbNtJ-5nUe66vry-oGc,18160
304
+ datahub/ingestion/source/dbt/dbt_common.py,sha256=QDttgzZQowsiZGq9EN7zGSbitUR17F3vo-JMkF8pJbw,82264
305
305
  datahub/ingestion/source/dbt/dbt_core.py,sha256=WVI2ZYXOMxgFzJnJqsqmEGS-5xdfiVIDsCb78lvSeQ0,24930
306
306
  datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpidfGTvOi5fr6I,9839
307
307
  datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -391,7 +391,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
391
391
  datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
392
392
  datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
393
393
  datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
394
- datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=PLQU1WnxDljZJGifoh6CqIQdBZrQAeBxtV-9NLdxM8o,15847
394
+ datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=qq05NhOWY1tB_Th528HT9WhVlhMdsIu5Cr9wmXvpkFw,17398
395
395
  datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
396
396
  datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
397
397
  datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -472,12 +472,12 @@ datahub/ingestion/source/snowflake/constants.py,sha256=XCW3vw4JfLn_s8-oXBX6WFNMP
472
472
  datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
473
473
  datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
474
474
  datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
475
- datahub/ingestion/source/snowflake/snowflake_config.py,sha256=z8HLbYY9U0eEMvMbEv249-aVfsTpfli72XVbErW39Eo,21460
475
+ datahub/ingestion/source/snowflake/snowflake_config.py,sha256=6uR7eZDmEfLpfs_QYNJGTTZI6N2VOvjqXKmG_7c6-9Q,22291
476
476
  datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUHYo_fvFp5VOAteCtR4GNjaUEvyNTNQ,18175
477
477
  datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
478
478
  datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
479
479
  datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
480
- datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=DWPs-ih4tNS7N5GftBkeh61zgL6UfKPXCY_RyspBaMA,41313
480
+ datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=deJwFf9Bg3bphMi6WNfPon59hFSLYJZ4ByQA4pUkTkU,42926
481
481
  datahub/ingestion/source/snowflake/snowflake_query.py,sha256=kqOxibplHyDhdioue8MeXBzeDS9d274-dspb-wyWMMI,38165
482
482
  datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
483
483
  datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
@@ -487,10 +487,10 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GK
487
487
  datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
488
488
  datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
489
489
  datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=w-SftpjHSOf_6S2T2srHJyJta3MHS0usmn4Z9jgx4QE,13858
490
- datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=gaHFtgdJYqAyQzVSOJ7I7CaZ3xMqRnWLH2YaDxISrgw,35062
490
+ datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=asZW8DztIB1TcGzOoZFmK6se22ZDaoG4L2NMvmd0yXY,35292
491
491
  datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=rOb78iHiWiK8v8WdVs1xDwVut4Y0OHmszej6IopQfCo,5341
492
492
  datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
493
- datahub/ingestion/source/sql/athena.py,sha256=lQ_zniHtmtSFCnSmzM2dG7ND6zOZmhu6t-I4ZOeQKho,27652
493
+ datahub/ingestion/source/sql/athena.py,sha256=Xh4UD-fAXP7qNrqmJ91fuNai3_21DXVOwKeGgOlgGqc,28067
494
494
  datahub/ingestion/source/sql/athena_properties_extractor.py,sha256=n2SvqeUbNWxiWWdkDs8VYlUPlLwfZzZy9AIa-V4D7AY,28531
495
495
  datahub/ingestion/source/sql/clickhouse.py,sha256=_uf-6fQYxI1-kMw0dNUvJf1f75XX0Qps4HMVlgzdpmo,25670
496
496
  datahub/ingestion/source/sql/cockroachdb.py,sha256=WoOKCq7YjsuzSPm1SmKIYZ9CrvlSF8zWmP1fNHn4G3Q,1360
@@ -500,7 +500,7 @@ datahub/ingestion/source/sql/hive.py,sha256=Sh3Us1vjUcbgQ4NxLKcXEdGtck5fJHIwOdjb
500
500
  datahub/ingestion/source/sql/hive_metastore.py,sha256=jpAgND_n2W6VCF_KMElXV5Gnd4-dTBQuFyjsAQUM9WI,35889
501
501
  datahub/ingestion/source/sql/mariadb.py,sha256=om6QoG5UtDldt1N6AfIWp3T-HXNaaqFmpz2i0JAemfM,654
502
502
  datahub/ingestion/source/sql/mysql.py,sha256=34Vu3otULxUY0-JUEYdZw2aoyuTlc9KLcoJdQxe2yJs,3267
503
- datahub/ingestion/source/sql/oracle.py,sha256=ftnrk3iiEelyv9PBHPYbairuP1WgxZbi1gu6YdqY69E,29965
503
+ datahub/ingestion/source/sql/oracle.py,sha256=BGrHln5OQ6gq7LTqY8e4ySS5-uJaJEan0TU1778ZEYs,29963
504
504
  datahub/ingestion/source/sql/postgres.py,sha256=t-28dTWLa3c_NgWlGS9ZVNFlZLxd6xTSguIKP4S4NHs,11887
505
505
  datahub/ingestion/source/sql/presto.py,sha256=tATa0M2q0PjUC_E9W_jSUsmKTP7cVJayLgrFMzG_eao,4223
506
506
  datahub/ingestion/source/sql/sql_common.py,sha256=qIH0Wnz8XvJDlYuS71X3XC5BfpeHkSGzpwn1aqPVY0o,52130
@@ -540,7 +540,7 @@ datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py
540
540
  datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
541
541
  datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
542
542
  datahub/ingestion/source/tableau/tableau.py,sha256=SO89gIsLUIxkHpTANUVdqkbsgpyO5mIcO1m3kZylgaM,155893
543
- datahub/ingestion/source/tableau/tableau_common.py,sha256=MIkHKZg_v2IVCRk-YdPlLZl3m0LcWZm5Indwb3IV2ZQ,26931
543
+ datahub/ingestion/source/tableau/tableau_common.py,sha256=4cUm3E8wLfjLSjcAXXWDWVUXAEho_hbsQa2BzAF-vtM,27012
544
544
  datahub/ingestion/source/tableau/tableau_constant.py,sha256=4ngrOwqxf4cgbLR3i0OKI4pUxmHMABKyywfhXQ0GazA,2592
545
545
  datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=nSyx9RzC6TCQDm-cTVJ657qT8iDwzk_8JMKpohhmOc4,1046
546
546
  datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
@@ -618,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
618
618
  datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
619
619
  datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
620
620
  datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
621
- datahub/metadata/_internal_schema_classes.py,sha256=DaI4G-ykM_OVmFJHwHVTPZ6fMk4z40TJ8bSmj5n_UNw,1043557
622
- datahub/metadata/schema.avsc,sha256=GInldoYJf7brWjnjY2qWv68vgfE_mOXVMCTvLkUzzmk,733204
621
+ datahub/metadata/_internal_schema_classes.py,sha256=Hp7LXq9qVe9KI1MDnIHgwaiJNbgbkI344AjfhwIUpBw,1044482
622
+ datahub/metadata/schema.avsc,sha256=0u6V_sIgWTFJldnsY9ZD7N_GyOyxufnAUsR_Yc_Nxws,733968
623
623
  datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
624
624
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
625
625
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -665,6 +665,7 @@ datahub/metadata/com/linkedin/pegasus2avro/glossary/__init__.py,sha256=fa1QNv08O
665
665
  datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py,sha256=EGxkzJgQMASL_aUmgjHE3bo8qRTSbAbM_8gUccZblX0,1603
666
666
  datahub/metadata/com/linkedin/pegasus2avro/incident/__init__.py,sha256=LfB7ytT1uuGl5Y8oeU6ILCpSGsGmUKgCjsHphn7AThg,916
667
667
  datahub/metadata/com/linkedin/pegasus2avro/ingestion/__init__.py,sha256=1bfG2naq4iS_pwU4J-BVer_gfL0hDbJbnH0gh1MPNgA,871
668
+ datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py,sha256=7SHiR-KzV1CkAimFy94SkcY0Xg0RlsIlLTUTGmGAW_U,290
668
669
  datahub/metadata/com/linkedin/pegasus2avro/metadata/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
669
670
  datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=G9CI1UqSXGzselvjnlOI7Obzjn5ZTQVzohRGBZHdnZk,5151
670
671
  datahub/metadata/com/linkedin/pegasus2avro/metadata/query/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
@@ -795,7 +796,7 @@ datahub/metadata/schemas/DataTypeKey.avsc,sha256=Gs5uc_azwg10e36ZbwDTFQMevr0IfiF
795
796
  datahub/metadata/schemas/DatahubIngestionCheckpoint.avsc,sha256=m2Zyrx3ZWDc5gHuwbmBSRJ3JN4NFkpUhDEKM2Yeuqrw,5681
796
797
  datahub/metadata/schemas/DatahubIngestionRunSummary.avsc,sha256=_Ek7NqfJVTLqlM0NR9BRA57N9_ejwDdQvz7B1tVxSEE,9367
797
798
  datahub/metadata/schemas/DatasetDeprecation.avsc,sha256=ucXxaDcAUib9_y0k5qOINMn5VK2X3trHK2dcpNcsR2Q,1256
798
- datahub/metadata/schemas/DatasetKey.avsc,sha256=xNh2Zbg1POuHD3qu42-__zNVhKWx6QkA2LY7Dlk0YSY,3473
799
+ datahub/metadata/schemas/DatasetKey.avsc,sha256=51gyLo-nOcAGchs9pGAfv2vCpWR2y0hN1Xr4A26ms94,3496
799
800
  datahub/metadata/schemas/DatasetProfile.avsc,sha256=3ZCU9JD6l2razACp0AY6LLMgnkMTj6D_5Xk9np6WWRM,9965
800
801
  datahub/metadata/schemas/DatasetProperties.avsc,sha256=DFJn75feqaoQk84zin_o_lqsFFhqkwya5LGC5LLJXbU,4209
801
802
  datahub/metadata/schemas/DatasetUpstreamLineage.avsc,sha256=PjAWPbsqwH7FjX2kFDy0dE6ENYOwRynH9vJerWisr2A,5365
@@ -858,6 +859,7 @@ datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=vNeAoFSIH1jEDa2rEyTStUj
858
859
  datahub/metadata/schemas/IntendedUse.avsc,sha256=IKZSWdvc0uAyyT-FtdQOGbMC-P7RS9cO0vOVKWT6fbw,1361
859
860
  datahub/metadata/schemas/InviteToken.avsc,sha256=8k_9MxHu9GVf7gvS0SlnQu7tJfpbXsRFdz6lQrFKPNc,737
860
861
  datahub/metadata/schemas/InviteTokenKey.avsc,sha256=MuQUlQaeVjaBkjSshB9gsx5Fm0civYgWD8UhCiRLdOQ,434
862
+ datahub/metadata/schemas/LogicalParent.avsc,sha256=U7PlYOsE3klr4ksbE_VEKKR2uQZ1KwXIhMXmipWcZa0,5159
861
863
  datahub/metadata/schemas/MLFeatureKey.avsc,sha256=1XFF8P2T3_4mX2oHGY74UJx1LFr6WtTv2jTG9ApjQoA,1150
862
864
  datahub/metadata/schemas/MLFeatureProperties.avsc,sha256=HpF7VcnH2FvDsqy2g5AMJPqU7upkTVADW1ps-9bjINo,6893
863
865
  datahub/metadata/schemas/MLFeatureTableKey.avsc,sha256=hVxNEqsx4GgG11GVryn9ms16OyEzsYdwM5RuAHy_454,1424
@@ -896,14 +898,14 @@ datahub/metadata/schemas/PostKey.avsc,sha256=MEA-S-RonGrsrpwYEejR6CXRka2DcqP7S7q
896
898
  datahub/metadata/schemas/QuantitativeAnalyses.avsc,sha256=XrGYVoGUh7ZRjAbMORSiqF5Zl4a0IbYDPiQuSJhABZM,960
897
899
  datahub/metadata/schemas/QueryKey.avsc,sha256=VI4oIHvAO7f0lN_7V3QVuBfHcPz31c57XtW6IrlDfxc,518
898
900
  datahub/metadata/schemas/QueryProperties.avsc,sha256=26Q3zzuzJbUCUG7IJ3q_OEdcNbyzloZzDJWPbv3GQAk,5589
899
- datahub/metadata/schemas/QuerySubjects.avsc,sha256=WbnUGiOBdc1Ypn4P_R5gCfPH-wNHJUIk7YeyJ2NiOZQ,1747
901
+ datahub/metadata/schemas/QuerySubjects.avsc,sha256=PDM6Ek1AkjwLGDk17Rjg7KVtE2tbgFpc1XTGkhoG4MA,1437
900
902
  datahub/metadata/schemas/QueryUsageStatistics.avsc,sha256=z1gfAnXdBoPEeERi5RESjrdBuS6AcIdqdN5JqWOSuNo,6192
901
903
  datahub/metadata/schemas/RoleKey.avsc,sha256=Uas5jFViSHXhFqq8D4P6-UXqywOth3coztjQ5wA7wL0,449
902
904
  datahub/metadata/schemas/RoleMembership.avsc,sha256=Al3LXKRowCiHhgTfwr3a-piID3Ld5kN-6-e9edTZ0uU,570
903
905
  datahub/metadata/schemas/RoleProperties.avsc,sha256=tDw-WF1uBGIcrk38nOnXs3FCF_YjBhScarJbreQvwjE,3037
904
906
  datahub/metadata/schemas/SchemaFieldAliases.avsc,sha256=El_cxn0KUhMf2LGfMPzcZ6Xtths2wQOaF9fnM1KQmxQ,560
905
907
  datahub/metadata/schemas/SchemaFieldInfo.avsc,sha256=Gf9EGqrEf10554hd4Eut7T8ZdOR-9OHgDXVRhFo311o,800
906
- datahub/metadata/schemas/SchemaFieldKey.avsc,sha256=DSNPsZftDVDQL9AnHOxa1arLKmQEG25xJ5VLTi15ueM,985
908
+ datahub/metadata/schemas/SchemaFieldKey.avsc,sha256=SnMq7HW-ctC22VVsaCb0Xdsr7S0FbpYN_5kvrjBujkY,1008
907
909
  datahub/metadata/schemas/SchemaMetadata.avsc,sha256=aOuIAYN_ISWQRhRSBNvM6wb3oQz6o_V-DnUMn4oOUPU,40994
908
910
  datahub/metadata/schemas/Siblings.avsc,sha256=lqzlNGP279TwFyFo7nUKx59atY2fyNGLlqIIJulSAz8,930
909
911
  datahub/metadata/schemas/SlackUserInfo.avsc,sha256=IY7InWaiDzJa3hJ9J4W3Eg8EUKuh0_gHG801FGQTB6o,3411
@@ -1091,8 +1093,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
1091
1093
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
1092
1094
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
1093
1095
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
1094
- acryl_datahub-1.1.0.5rc11.dist-info/METADATA,sha256=foPeHlIiZp3IZgq2Objhuchv_WwA6lfnBq7_Xnuzcgo,181840
1095
- acryl_datahub-1.1.0.5rc11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1096
- acryl_datahub-1.1.0.5rc11.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
1097
- acryl_datahub-1.1.0.5rc11.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1098
- acryl_datahub-1.1.0.5rc11.dist-info/RECORD,,
1096
+ acryl_datahub-1.1.0.5rc13.dist-info/METADATA,sha256=K0_AWqbWSWduRqBlxLCjnGD-7BQM1JYYH8y7dwzM6yE,181896
1097
+ acryl_datahub-1.1.0.5rc13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1098
+ acryl_datahub-1.1.0.5rc13.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
1099
+ acryl_datahub-1.1.0.5rc13.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1100
+ acryl_datahub-1.1.0.5rc13.dist-info/RECORD,,
datahub/_version.py CHANGED
@@ -1,6 +1,6 @@
1
1
  # Published at https://pypi.org/project/acryl-datahub/.
2
2
  __package_name__ = "acryl-datahub"
3
- __version__ = "1.1.0.5rc11"
3
+ __version__ = "1.1.0.5rc13"
4
4
 
5
5
 
6
6
  def is_dev_mode() -> bool:
@@ -162,12 +162,10 @@ class QuickstartVersionMappingConfig(BaseModel):
162
162
  # docker compose based resolved compose file. In those cases, we pick up the composefile from
163
163
  # MINIMUM_SUPPORTED_VERSION which contains the compose file.
164
164
  if _is_it_a_version(result.composefile_git_ref):
165
- if (
166
- parse("v1.2.0") > parse(result.composefile_git_ref)
167
- ): # TODO: Once we decide what the new version that supports profile based compose.
168
- # The merge commit where the labels were added
169
- # https://github.com/datahub-project/datahub/pull/7473
170
- result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e" # TODO update after compose file is merged
165
+ if parse("v1.2.0") > parse(result.composefile_git_ref):
166
+ # The merge commit where profiles based resolved compose file was added.
167
+ # https://github.com/datahub-project/datahub/pull/13566
168
+ result.composefile_git_ref = "21726bc3341490f4182b904626c793091ac95edd"
171
169
 
172
170
  return result
173
171
 
@@ -230,9 +230,10 @@ class SourceReport(ExamplesReport):
230
230
  context: Optional[str] = None,
231
231
  title: Optional[LiteralString] = None,
232
232
  exc: Optional[BaseException] = None,
233
+ log: bool = True,
233
234
  ) -> None:
234
235
  self._structured_logs.report_log(
235
- StructuredLogLevel.WARN, message, title, context, exc, log=True
236
+ StructuredLogLevel.WARN, message, title, context, exc, log=log
236
237
  )
237
238
 
238
239
  def report_failure(
@@ -533,7 +533,7 @@ class ABSSource(StatefulIngestionSourceBase):
533
533
  )
534
534
  path_spec.sample_files = False
535
535
  for obj in container_client.list_blobs(
536
- prefix=f"{prefix}", results_per_page=PAGE_SIZE
536
+ name_starts_with=f"{prefix}", results_per_page=PAGE_SIZE
537
537
  ):
538
538
  abs_path = self.create_abs_path(obj.name)
539
539
  logger.debug(f"Path: {abs_path}")
@@ -61,13 +61,13 @@ class AzureConnectionConfig(ConfigModel):
61
61
  def get_blob_service_client(self):
62
62
  return BlobServiceClient(
63
63
  account_url=f"https://{self.account_name}.blob.core.windows.net",
64
- credential=f"{self.get_credentials()}",
64
+ credential=self.get_credentials(),
65
65
  )
66
66
 
67
67
  def get_data_lake_service_client(self) -> DataLakeServiceClient:
68
68
  return DataLakeServiceClient(
69
69
  account_url=f"https://{self.account_name}.dfs.core.windows.net",
70
- credential=f"{self.get_credentials()}",
70
+ credential=self.get_credentials(),
71
71
  )
72
72
 
73
73
  def get_credentials(
@@ -342,7 +342,7 @@ class BigQueryV2Config(
342
342
  )
343
343
 
344
344
  use_queries_v2: bool = Field(
345
- default=False,
345
+ default=True,
346
346
  description="If enabled, uses the new queries extractor to extract queries from bigquery.",
347
347
  )
348
348
  include_queries: bool = Field(
@@ -1,3 +1,4 @@
1
+ import re
1
2
  from abc import ABC, abstractmethod
2
3
 
3
4
  # Add imports for source customization
@@ -236,42 +237,76 @@ class ABSObjectStore(ObjectStoreInterface):
236
237
  """Implementation of ObjectStoreInterface for Azure Blob Storage."""
237
238
 
238
239
  PREFIX = "abfss://"
240
+ HTTPS_REGEX = re.compile(r"(https?://[a-z0-9]{3,24}\.blob\.core\.windows\.net/)")
239
241
 
240
242
  @classmethod
241
243
  def is_uri(cls, uri: str) -> bool:
242
- return uri.startswith(cls.PREFIX)
244
+ return uri.startswith(cls.PREFIX) or bool(cls.HTTPS_REGEX.match(uri))
243
245
 
244
246
  @classmethod
245
247
  def get_prefix(cls, uri: str) -> Optional[str]:
246
248
  if uri.startswith(cls.PREFIX):
247
249
  return cls.PREFIX
250
+
251
+ # Check for HTTPS format
252
+ match = cls.HTTPS_REGEX.match(uri)
253
+ if match:
254
+ return match.group(1)
255
+
248
256
  return None
249
257
 
250
258
  @classmethod
251
259
  def strip_prefix(cls, uri: str) -> str:
252
- prefix = cls.get_prefix(uri)
253
- if not prefix:
254
- raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
255
- return uri[len(prefix) :]
260
+ if uri.startswith(cls.PREFIX):
261
+ return uri[len(cls.PREFIX) :]
262
+
263
+ # Handle HTTPS format
264
+ match = cls.HTTPS_REGEX.match(uri)
265
+ if match:
266
+ return uri[len(match.group(1)) :]
267
+
268
+ raise ValueError(
269
+ f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
270
+ )
256
271
 
257
272
  @classmethod
258
273
  def get_bucket_name(cls, uri: str) -> str:
259
274
  if not cls.is_uri(uri):
260
- raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
261
- return cls.strip_prefix(uri).split("@")[0]
275
+ raise ValueError(
276
+ f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
277
+ )
278
+
279
+ if uri.startswith(cls.PREFIX):
280
+ # abfss://container@account.dfs.core.windows.net/path
281
+ return cls.strip_prefix(uri).split("@")[0]
282
+ else:
283
+ # https://account.blob.core.windows.net/container/path
284
+ return cls.strip_prefix(uri).split("/")[0]
262
285
 
263
286
  @classmethod
264
287
  def get_object_key(cls, uri: str) -> str:
265
288
  if not cls.is_uri(uri):
266
- raise ValueError(f"Not an ABS URI. Must start with prefix: {cls.PREFIX}")
267
- parts = cls.strip_prefix(uri).split("@", 1)
268
- if len(parts) < 2:
269
- return ""
270
- account_path = parts[1]
271
- path_parts = account_path.split("/", 1)
272
- if len(path_parts) < 2:
273
- return ""
274
- return path_parts[1]
289
+ raise ValueError(
290
+ f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
291
+ )
292
+
293
+ if uri.startswith(cls.PREFIX):
294
+ # abfss://container@account.dfs.core.windows.net/path
295
+ parts = cls.strip_prefix(uri).split("@", 1)
296
+ if len(parts) < 2:
297
+ return ""
298
+ account_path = parts[1]
299
+ path_parts = account_path.split("/", 1)
300
+ if len(path_parts) < 2:
301
+ return ""
302
+ return path_parts[1]
303
+ else:
304
+ # https://account.blob.core.windows.net/container/path
305
+ stripped = cls.strip_prefix(uri)
306
+ parts = stripped.split("/", 1)
307
+ if len(parts) < 2:
308
+ return ""
309
+ return parts[1]
275
310
 
276
311
 
277
312
  # Registry of all object store implementations
@@ -331,6 +366,12 @@ def get_object_store_bucket_name(uri: str) -> str:
331
366
  return uri[prefix_length:].split("/")[0]
332
367
  elif uri.startswith(ABSObjectStore.PREFIX):
333
368
  return uri[len(ABSObjectStore.PREFIX) :].split("@")[0]
369
+ elif ABSObjectStore.HTTPS_REGEX.match(uri):
370
+ # Handle HTTPS Azure Blob Storage URLs
371
+ match = ABSObjectStore.HTTPS_REGEX.match(uri)
372
+ if match:
373
+ stripped = uri[len(match.group(1)) :]
374
+ return stripped.split("/")[0]
334
375
 
335
376
  raise ValueError(f"Unsupported URI format: {uri}")
336
377
 
@@ -470,18 +511,25 @@ class ObjectStoreSourceAdapter:
470
511
  if not ABSObjectStore.is_uri(table_data.table_path):
471
512
  return None
472
513
 
473
- # Parse the ABS URI
474
514
  try:
475
- # URI format: abfss://container@account.dfs.core.windows.net/path
476
- path_without_prefix = ABSObjectStore.strip_prefix(table_data.table_path)
477
- parts = path_without_prefix.split("@", 1)
478
- if len(parts) < 2:
479
- return None
480
-
481
- container_name = parts[0]
482
- account_parts = parts[1].split("/", 1)
483
- account_domain = account_parts[0]
484
- account_name = account_domain.split(".")[0]
515
+ if table_data.table_path.startswith("abfss://"):
516
+ # URI format: abfss://container@account.dfs.core.windows.net/path
517
+ path_without_prefix = ABSObjectStore.strip_prefix(table_data.table_path)
518
+ parts = path_without_prefix.split("@", 1)
519
+ if len(parts) < 2:
520
+ return None
521
+
522
+ container_name = parts[0]
523
+ account_parts = parts[1].split("/", 1)
524
+ account_domain = account_parts[0]
525
+ account_name = account_domain.split(".")[0]
526
+ else:
527
+ # Handle HTTPS format: https://account.blob.core.windows.net/container/path
528
+ container_name = ABSObjectStore.get_bucket_name(table_data.table_path)
529
+ if "blob.core.windows.net" in table_data.table_path:
530
+ account_name = table_data.table_path.split("//")[1].split(".")[0]
531
+ else:
532
+ return None
485
533
 
486
534
  # Construct Azure portal URL
487
535
  return f"https://portal.azure.com/#blade/Microsoft_Azure_Storage/ContainerMenuBlade/overview/storageAccountId/{account_name}/containerName/{container_name}"
@@ -26,6 +26,7 @@ from datahub.ingestion.source.dbt.dbt_common import (
26
26
  DBTCommonConfig,
27
27
  DBTNode,
28
28
  DBTSourceBase,
29
+ DBTSourceReport,
29
30
  )
30
31
  from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
31
32
 
@@ -266,6 +267,7 @@ query DatahubMetadataQuery_{type}($jobId: BigInt!, $runId: BigInt) {{
266
267
  @capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
267
268
  class DBTCloudSource(DBTSourceBase, TestableSource):
268
269
  config: DBTCloudConfig
270
+ report: DBTSourceReport # nothing cloud-specific in the report
269
271
 
270
272
  @classmethod
271
273
  def create(cls, config_dict, ctx):
@@ -404,8 +406,11 @@ class DBTCloudSource(DBTSourceBase, TestableSource):
404
406
  if node["resourceType"] in {"model", "seed", "snapshot"}:
405
407
  status = node["status"]
406
408
  if status is None and materialization != "ephemeral":
407
- self.report.report_warning(
408
- key, "node is missing a status, schema metadata will be incomplete"
409
+ self.report.warning(
410
+ title="Schema information may be incomplete",
411
+ message="Some nodes are missing the `status` field, which dbt uses to track the status of the node in the target database.",
412
+ context=key,
413
+ log=False,
409
414
  )
410
415
 
411
416
  # The code fields are new in dbt 1.3, and replace the sql ones.
@@ -1336,6 +1336,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
1336
1336
  self.config.tag_prefix,
1337
1337
  "SOURCE_CONTROL",
1338
1338
  self.config.strip_user_ids_from_email,
1339
+ match_nested_props=True,
1339
1340
  )
1340
1341
 
1341
1342
  action_processor_tag = OperationProcessor(
@@ -1707,6 +1708,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
1707
1708
  self.config.tag_prefix,
1708
1709
  "SOURCE_CONTROL",
1709
1710
  self.config.strip_user_ids_from_email,
1711
+ match_nested_props=True,
1710
1712
  )
1711
1713
 
1712
1714
  canonical_schema: List[SchemaField] = []
@@ -70,10 +70,19 @@ class LineageConfigGen1(ConfigModel):
70
70
  Table naming convention: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
71
71
  """
72
72
 
73
- emit_lineage: bool = Field(
73
+ enabled: bool = Field(
74
74
  default=False,
75
+ description="Whether this source is enabled",
76
+ )
77
+
78
+ emit_lineage: bool = Field(
79
+ default=True,
75
80
  description="Whether to emit lineage data for testing purposes. When False, no lineage data is generated regardless of other settings.",
76
81
  )
82
+ emit_usage: bool = Field(
83
+ default=True,
84
+ description="Whether to emit usage data for testing purposes. When False, no usage data is generated regardless of other settings.",
85
+ )
77
86
 
78
87
  lineage_fan_out: int = Field(
79
88
  default=3,
@@ -106,6 +115,18 @@ class DataHubMockDataConfig(ConfigModel):
106
115
  default=True,
107
116
  description="Whether this source is enabled",
108
117
  )
118
+ throw_uncaught_exceptions: bool = Field(
119
+ default=False,
120
+ description="Whether to throw an uncaught exception for testing",
121
+ )
122
+ num_errors: int = Field(
123
+ default=0,
124
+ description="Number of errors to add in report for testing",
125
+ )
126
+ num_warnings: int = Field(
127
+ default=0,
128
+ description="Number of warnings to add in report for testing",
129
+ )
109
130
 
110
131
  gen_1: LineageConfigGen1 = Field(
111
132
  default_factory=LineageConfigGen1,
@@ -128,9 +149,28 @@ class DataHubMockDataSource(Source):
128
149
  self.report = DataHubMockDataReport()
129
150
 
130
151
  def get_workunits(self) -> Iterable[MetadataWorkUnit]:
152
+ if self.config.throw_uncaught_exceptions:
153
+ raise Exception("This is a test exception")
154
+
155
+ if self.config.num_errors > 0:
156
+ for i in range(self.config.num_errors):
157
+ self.report.failure(
158
+ message="This is test error message",
159
+ title="Test Error",
160
+ context=f"This is test error {i}",
161
+ )
162
+
163
+ if self.config.num_warnings > 0:
164
+ for i in range(self.config.num_warnings):
165
+ self.report.warning(
166
+ message="This is test warning",
167
+ title="Test Warning",
168
+ context=f"This is test warning {i}",
169
+ )
170
+
131
171
  # We don't want any implicit aspects to be produced
132
172
  # so we are not using get_workunits_internal
133
- if self.config.gen_1.emit_lineage:
173
+ if self.config.gen_1.enabled:
134
174
  for wu in self._data_gen_1():
135
175
  if self.report.first_urn_seen is None:
136
176
  self.report.first_urn_seen = wu.get_urn()
@@ -284,17 +324,19 @@ class DataHubMockDataSource(Source):
284
324
 
285
325
  yield self._get_profile_aspect(table_name)
286
326
 
287
- yield self._get_usage_aspect(table_name)
288
-
289
- yield from self._generate_lineage_for_table(
290
- table_name=table_name,
291
- table_level=i,
292
- table_index=j,
293
- hops=hops,
294
- fan_out=fan_out,
295
- fan_out_after_first=fan_out_after_first,
296
- tables_at_levels=tables_at_levels,
297
- )
327
+ if self.config.gen_1.emit_usage:
328
+ yield self._get_usage_aspect(table_name)
329
+
330
+ if self.config.gen_1.emit_lineage:
331
+ yield from self._generate_lineage_for_table(
332
+ table_name=table_name,
333
+ table_level=i,
334
+ table_index=j,
335
+ hops=hops,
336
+ fan_out=fan_out,
337
+ fan_out_after_first=fan_out_after_first,
338
+ tables_at_levels=tables_at_levels,
339
+ )
298
340
 
299
341
  def _generate_lineage_for_table(
300
342
  self,
@@ -236,7 +236,7 @@ class SnowflakeV2Config(
236
236
  )
237
237
 
238
238
  use_queries_v2: bool = Field(
239
- default=False,
239
+ default=True,
240
240
  description="If enabled, uses the new queries extractor to extract queries from snowflake.",
241
241
  )
242
242
  include_queries: bool = Field(
@@ -361,6 +361,20 @@ class SnowflakeV2Config(
361
361
  "Only applicable if `use_queries_v2` is enabled.",
362
362
  )
363
363
 
364
+ push_down_database_pattern_access_history: bool = Field(
365
+ default=False,
366
+ description="If enabled, pushes down database pattern filtering to the access_history table for improved performance. "
367
+ "This filters on the accessed objects in access_history.",
368
+ )
369
+
370
+ additional_database_names_allowlist: List[str] = Field(
371
+ default=[],
372
+ description="Additional database names (no pattern matching) to be included in the access_history filter. "
373
+ "Only applies if push_down_database_pattern_access_history=True. "
374
+ "These databases will be included in the filter being pushed down regardless of database_pattern settings."
375
+ "This may be required in the case of _eg_ temporary tables being created in a different database than the ones in the database_name patterns.",
376
+ )
377
+
364
378
  @validator("convert_urns_to_lowercase")
365
379
  def validate_convert_urns_to_lowercase(cls, v):
366
380
  if not v: