acryl-datahub 1.1.0.5rc11__py3-none-any.whl → 1.1.0.5rc13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/METADATA +2593 -2593
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/RECORD +29 -27
- datahub/_version.py +1 -1
- datahub/cli/quickstart_versioning.py +4 -6
- datahub/ingestion/api/source.py +2 -1
- datahub/ingestion/source/abs/source.py +1 -1
- datahub/ingestion/source/azure/azure_common.py +2 -2
- datahub/ingestion/source/bigquery_v2/bigquery_config.py +1 -1
- datahub/ingestion/source/data_lake_common/object_store.py +75 -27
- datahub/ingestion/source/dbt/dbt_cloud.py +7 -2
- datahub/ingestion/source/dbt/dbt_common.py +2 -0
- datahub/ingestion/source/mock_data/datahub_mock_data.py +55 -13
- datahub/ingestion/source/snowflake/snowflake_config.py +15 -1
- datahub/ingestion/source/snowflake/snowflake_queries.py +57 -19
- datahub/ingestion/source/snowflake/snowflake_v2.py +2 -0
- datahub/ingestion/source/sql/athena.py +10 -0
- datahub/ingestion/source/sql/oracle.py +1 -1
- datahub/ingestion/source/tableau/tableau_common.py +4 -2
- datahub/metadata/_internal_schema_classes.py +35 -2
- datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
- datahub/metadata/schema.avsc +38 -11
- datahub/metadata/schemas/DatasetKey.avsc +2 -1
- datahub/metadata/schemas/LogicalParent.avsc +140 -0
- datahub/metadata/schemas/QuerySubjects.avsc +1 -12
- datahub/metadata/schemas/SchemaFieldKey.avsc +2 -1
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.1.0.5rc11.dist-info → acryl_datahub-1.1.0.5rc13.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.1.0.
|
|
1
|
+
acryl_datahub-1.1.0.5rc13.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=AWm-T7Sf9DTy4rZWQzbDvsJbdT1nZJ1ux45nnicW6tI,324
|
|
5
5
|
datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -84,7 +84,7 @@ datahub/cli/lite_cli.py,sha256=XKMejSuYUToKBvgN3YmmnxjRcaG5WPw23gJuQK8pgRc,13099
|
|
|
84
84
|
datahub/cli/migrate.py,sha256=XWW_fd4EUQGAUrKh9oKmtNYmfhcvbqWZJj3TQNgD8Dk,18012
|
|
85
85
|
datahub/cli/migration_utils.py,sha256=snkkT8ixo3Il4bvc7qN_QBOP1fz5UsO0usrUl2zJ68w,9478
|
|
86
86
|
datahub/cli/put_cli.py,sha256=ehqsxEPVVK-0xu7nVfBl_GxfsDbkgjIjkF8mfGfcIpc,3827
|
|
87
|
-
datahub/cli/quickstart_versioning.py,sha256=
|
|
87
|
+
datahub/cli/quickstart_versioning.py,sha256=Ec7SehEImRQW4I_8O89cTG1MvZtnnd8EJx9X8NaauoY,7302
|
|
88
88
|
datahub/cli/state_cli.py,sha256=RDXYCb17sqjW49nNdkoGAqNPRRHOKw5lcgcs5QvHeOM,1142
|
|
89
89
|
datahub/cli/telemetry.py,sha256=xw3SiAn2je48Qv4kXPYN5EPVKHWEWZc2LGoF7UzGs8U,489
|
|
90
90
|
datahub/cli/timeline_cli.py,sha256=Wc_Mo7khFFHXgRSl2FgtvB8m0QhuCEhkpaFCIOPpHJE,7584
|
|
@@ -151,7 +151,7 @@ datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX
|
|
|
151
151
|
datahub/ingestion/api/report.py,sha256=_bP7JPHZyftWnCJwFs6YId21mQ0-2rd406H91lLHW98,16942
|
|
152
152
|
datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
|
|
153
153
|
datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
|
|
154
|
-
datahub/ingestion/api/source.py,sha256=
|
|
154
|
+
datahub/ingestion/api/source.py,sha256=1vZu4oWLS1D8XMV55UjxOkOnMSQjnx0NMwVFFDS065Y,19694
|
|
155
155
|
datahub/ingestion/api/source_helpers.py,sha256=XT9y5HgfVeF52jrX39vlLn1SdXpLVyT2Su8oGNsddYo,21148
|
|
156
156
|
datahub/ingestion/api/transform.py,sha256=X0GpjMJzYkLuZx8MTWxH50cWGm9rGsnn3k188mmC8J8,582
|
|
157
157
|
datahub/ingestion/api/workunit.py,sha256=e8n8RfSjHZZm2R4ShNH0UuMtUkMjyqqM2j2t7oL74lo,6327
|
|
@@ -234,7 +234,7 @@ datahub/ingestion/source/abs/config.py,sha256=mBQe0JTaP-Rcv4HnMUUySoYbSr4r3jDEMi
|
|
|
234
234
|
datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=Rkf64evufyVGPiE4VK8QAjzBiJFu85tOGMmJ0lJZ2Og,3600
|
|
235
235
|
datahub/ingestion/source/abs/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
|
|
236
236
|
datahub/ingestion/source/abs/report.py,sha256=CkRjsNn0Pab-ZPllxz3IUJI_r3x0T6urJePa_hJKi5U,586
|
|
237
|
-
datahub/ingestion/source/abs/source.py,sha256=
|
|
237
|
+
datahub/ingestion/source/abs/source.py,sha256=FvncxJG2ZjQ4Q5JeYo4zVYuqEBsfvHOP4ZSjy_CKG90,24826
|
|
238
238
|
datahub/ingestion/source/apply/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
239
239
|
datahub/ingestion/source/apply/datahub_apply.py,sha256=xTD-Iq3UHhxcz61RwNuI2kJjRrnQEfZFSgvS1X6loV4,7703
|
|
240
240
|
datahub/ingestion/source/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -254,12 +254,12 @@ datahub/ingestion/source/aws/sagemaker_processors/models.py,sha256=N9FtnOznwwToc
|
|
|
254
254
|
datahub/ingestion/source/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
255
255
|
datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0ch1oF2SJSYDZ1JMB_Onso,7605
|
|
256
256
|
datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
|
|
257
|
-
datahub/ingestion/source/azure/azure_common.py,sha256=
|
|
257
|
+
datahub/ingestion/source/azure/azure_common.py,sha256=jr9i0m__TuGJpEQ5BwHrQw1M9s8j_OEn-iZ8DPX6cuk,4043
|
|
258
258
|
datahub/ingestion/source/bigquery_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
259
259
|
datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=u4-LLt6ZDe3hKqLWqEByYpc0z-UcEZf85uok9qNEFko,15321
|
|
260
260
|
datahub/ingestion/source/bigquery_v2/bigquery_audit.py,sha256=kEwWhq3ch6WT4q4hcX8-fvQh28KgrNfspFwIytO3vQA,25103
|
|
261
261
|
datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py,sha256=LuGJ6LgPViLIfDQfylxlQ3CA7fZYM5MDt8M-7sfzm84,5096
|
|
262
|
-
datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=
|
|
262
|
+
datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=DYfZQIwcB9kzf0Kv_m5sUhdBYtrl4LTriZZ7n7mK4-w,21915
|
|
263
263
|
datahub/ingestion/source/bigquery_v2/bigquery_connection.py,sha256=sq9Nk11W5cMPs4X9ST6OBb3UnHkSeyBHtB_bKU1lOj8,2799
|
|
264
264
|
datahub/ingestion/source/bigquery_v2/bigquery_data_reader.py,sha256=DeT3v_Z82__8En0FcZ0kavBAWQoRvSZ5Rppm9eeDAb8,2393
|
|
265
265
|
datahub/ingestion/source/bigquery_v2/bigquery_helper.py,sha256=QER3gY8e_k1_eNVj7cBso7ZzrWl_vO5PYSa6CpvqNx8,1554
|
|
@@ -289,7 +289,7 @@ datahub/ingestion/source/common/subtypes.py,sha256=x8dv9SN23Y8t5oq7SUuQWUPAH8KM3
|
|
|
289
289
|
datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
290
290
|
datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
|
|
291
291
|
datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGDTUOol9Oi3jZs8Pb04PIsHdC2WPP0dkg,6576
|
|
292
|
-
datahub/ingestion/source/data_lake_common/object_store.py,sha256=
|
|
292
|
+
datahub/ingestion/source/data_lake_common/object_store.py,sha256=i9Hgb8Ww23QD_jEjzj_2qxA8Nr56krnZfo1qyOWmH9M,23608
|
|
293
293
|
datahub/ingestion/source/data_lake_common/path_spec.py,sha256=ekJAr4-PE2RhzQnmKb2xcSs_YncC1Dz95-UrXI67Vos,23584
|
|
294
294
|
datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
295
295
|
datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
|
|
@@ -300,8 +300,8 @@ datahub/ingestion/source/datahub/datahub_source.py,sha256=LsfDntDtqt_HMYCWwYctFK
|
|
|
300
300
|
datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vBCU0XxGcZR6Xxs,940
|
|
301
301
|
datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
|
|
302
302
|
datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
303
|
-
datahub/ingestion/source/dbt/dbt_cloud.py,sha256=
|
|
304
|
-
datahub/ingestion/source/dbt/dbt_common.py,sha256=
|
|
303
|
+
datahub/ingestion/source/dbt/dbt_cloud.py,sha256=4gWOFSX0YU8EAJgO4J47NBE4QbNtJ-5nUe66vry-oGc,18160
|
|
304
|
+
datahub/ingestion/source/dbt/dbt_common.py,sha256=QDttgzZQowsiZGq9EN7zGSbitUR17F3vo-JMkF8pJbw,82264
|
|
305
305
|
datahub/ingestion/source/dbt/dbt_core.py,sha256=WVI2ZYXOMxgFzJnJqsqmEGS-5xdfiVIDsCb78lvSeQ0,24930
|
|
306
306
|
datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpidfGTvOi5fr6I,9839
|
|
307
307
|
datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -391,7 +391,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
|
|
|
391
391
|
datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
|
|
392
392
|
datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
|
|
393
393
|
datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
394
|
-
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=
|
|
394
|
+
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=qq05NhOWY1tB_Th528HT9WhVlhMdsIu5Cr9wmXvpkFw,17398
|
|
395
395
|
datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
|
|
396
396
|
datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
|
|
397
397
|
datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -472,12 +472,12 @@ datahub/ingestion/source/snowflake/constants.py,sha256=XCW3vw4JfLn_s8-oXBX6WFNMP
|
|
|
472
472
|
datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
|
|
473
473
|
datahub/ingestion/source/snowflake/oauth_generator.py,sha256=fu2VnREGuJXeTqIV2jx4TwieVnznf83HQkrE0h2DGGM,3423
|
|
474
474
|
datahub/ingestion/source/snowflake/snowflake_assertion.py,sha256=_l3k4aI9wvioE81xxdeizJn9nJCZ_nMIXgk9N6pEk5o,4803
|
|
475
|
-
datahub/ingestion/source/snowflake/snowflake_config.py,sha256=
|
|
475
|
+
datahub/ingestion/source/snowflake/snowflake_config.py,sha256=6uR7eZDmEfLpfs_QYNJGTTZI6N2VOvjqXKmG_7c6-9Q,22291
|
|
476
476
|
datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUHYo_fvFp5VOAteCtR4GNjaUEvyNTNQ,18175
|
|
477
477
|
datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
|
|
478
478
|
datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
|
|
479
479
|
datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
|
|
480
|
-
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=
|
|
480
|
+
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=deJwFf9Bg3bphMi6WNfPon59hFSLYJZ4ByQA4pUkTkU,42926
|
|
481
481
|
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=kqOxibplHyDhdioue8MeXBzeDS9d274-dspb-wyWMMI,38165
|
|
482
482
|
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
|
|
483
483
|
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
|
|
@@ -487,10 +487,10 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GK
|
|
|
487
487
|
datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
|
|
488
488
|
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
|
|
489
489
|
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=w-SftpjHSOf_6S2T2srHJyJta3MHS0usmn4Z9jgx4QE,13858
|
|
490
|
-
datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=
|
|
490
|
+
datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=asZW8DztIB1TcGzOoZFmK6se22ZDaoG4L2NMvmd0yXY,35292
|
|
491
491
|
datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=rOb78iHiWiK8v8WdVs1xDwVut4Y0OHmszej6IopQfCo,5341
|
|
492
492
|
datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
493
|
-
datahub/ingestion/source/sql/athena.py,sha256=
|
|
493
|
+
datahub/ingestion/source/sql/athena.py,sha256=Xh4UD-fAXP7qNrqmJ91fuNai3_21DXVOwKeGgOlgGqc,28067
|
|
494
494
|
datahub/ingestion/source/sql/athena_properties_extractor.py,sha256=n2SvqeUbNWxiWWdkDs8VYlUPlLwfZzZy9AIa-V4D7AY,28531
|
|
495
495
|
datahub/ingestion/source/sql/clickhouse.py,sha256=_uf-6fQYxI1-kMw0dNUvJf1f75XX0Qps4HMVlgzdpmo,25670
|
|
496
496
|
datahub/ingestion/source/sql/cockroachdb.py,sha256=WoOKCq7YjsuzSPm1SmKIYZ9CrvlSF8zWmP1fNHn4G3Q,1360
|
|
@@ -500,7 +500,7 @@ datahub/ingestion/source/sql/hive.py,sha256=Sh3Us1vjUcbgQ4NxLKcXEdGtck5fJHIwOdjb
|
|
|
500
500
|
datahub/ingestion/source/sql/hive_metastore.py,sha256=jpAgND_n2W6VCF_KMElXV5Gnd4-dTBQuFyjsAQUM9WI,35889
|
|
501
501
|
datahub/ingestion/source/sql/mariadb.py,sha256=om6QoG5UtDldt1N6AfIWp3T-HXNaaqFmpz2i0JAemfM,654
|
|
502
502
|
datahub/ingestion/source/sql/mysql.py,sha256=34Vu3otULxUY0-JUEYdZw2aoyuTlc9KLcoJdQxe2yJs,3267
|
|
503
|
-
datahub/ingestion/source/sql/oracle.py,sha256=
|
|
503
|
+
datahub/ingestion/source/sql/oracle.py,sha256=BGrHln5OQ6gq7LTqY8e4ySS5-uJaJEan0TU1778ZEYs,29963
|
|
504
504
|
datahub/ingestion/source/sql/postgres.py,sha256=t-28dTWLa3c_NgWlGS9ZVNFlZLxd6xTSguIKP4S4NHs,11887
|
|
505
505
|
datahub/ingestion/source/sql/presto.py,sha256=tATa0M2q0PjUC_E9W_jSUsmKTP7cVJayLgrFMzG_eao,4223
|
|
506
506
|
datahub/ingestion/source/sql/sql_common.py,sha256=qIH0Wnz8XvJDlYuS71X3XC5BfpeHkSGzpwn1aqPVY0o,52130
|
|
@@ -540,7 +540,7 @@ datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py
|
|
|
540
540
|
datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
|
|
541
541
|
datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
542
542
|
datahub/ingestion/source/tableau/tableau.py,sha256=SO89gIsLUIxkHpTANUVdqkbsgpyO5mIcO1m3kZylgaM,155893
|
|
543
|
-
datahub/ingestion/source/tableau/tableau_common.py,sha256=
|
|
543
|
+
datahub/ingestion/source/tableau/tableau_common.py,sha256=4cUm3E8wLfjLSjcAXXWDWVUXAEho_hbsQa2BzAF-vtM,27012
|
|
544
544
|
datahub/ingestion/source/tableau/tableau_constant.py,sha256=4ngrOwqxf4cgbLR3i0OKI4pUxmHMABKyywfhXQ0GazA,2592
|
|
545
545
|
datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=nSyx9RzC6TCQDm-cTVJ657qT8iDwzk_8JMKpohhmOc4,1046
|
|
546
546
|
datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
|
|
@@ -618,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
|
|
|
618
618
|
datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
|
|
619
619
|
datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
|
|
620
620
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
621
|
-
datahub/metadata/_internal_schema_classes.py,sha256=
|
|
622
|
-
datahub/metadata/schema.avsc,sha256=
|
|
621
|
+
datahub/metadata/_internal_schema_classes.py,sha256=Hp7LXq9qVe9KI1MDnIHgwaiJNbgbkI344AjfhwIUpBw,1044482
|
|
622
|
+
datahub/metadata/schema.avsc,sha256=0u6V_sIgWTFJldnsY9ZD7N_GyOyxufnAUsR_Yc_Nxws,733968
|
|
623
623
|
datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
|
|
624
624
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
625
625
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -665,6 +665,7 @@ datahub/metadata/com/linkedin/pegasus2avro/glossary/__init__.py,sha256=fa1QNv08O
|
|
|
665
665
|
datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py,sha256=EGxkzJgQMASL_aUmgjHE3bo8qRTSbAbM_8gUccZblX0,1603
|
|
666
666
|
datahub/metadata/com/linkedin/pegasus2avro/incident/__init__.py,sha256=LfB7ytT1uuGl5Y8oeU6ILCpSGsGmUKgCjsHphn7AThg,916
|
|
667
667
|
datahub/metadata/com/linkedin/pegasus2avro/ingestion/__init__.py,sha256=1bfG2naq4iS_pwU4J-BVer_gfL0hDbJbnH0gh1MPNgA,871
|
|
668
|
+
datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py,sha256=7SHiR-KzV1CkAimFy94SkcY0Xg0RlsIlLTUTGmGAW_U,290
|
|
668
669
|
datahub/metadata/com/linkedin/pegasus2avro/metadata/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
|
|
669
670
|
datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=G9CI1UqSXGzselvjnlOI7Obzjn5ZTQVzohRGBZHdnZk,5151
|
|
670
671
|
datahub/metadata/com/linkedin/pegasus2avro/metadata/query/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
|
|
@@ -795,7 +796,7 @@ datahub/metadata/schemas/DataTypeKey.avsc,sha256=Gs5uc_azwg10e36ZbwDTFQMevr0IfiF
|
|
|
795
796
|
datahub/metadata/schemas/DatahubIngestionCheckpoint.avsc,sha256=m2Zyrx3ZWDc5gHuwbmBSRJ3JN4NFkpUhDEKM2Yeuqrw,5681
|
|
796
797
|
datahub/metadata/schemas/DatahubIngestionRunSummary.avsc,sha256=_Ek7NqfJVTLqlM0NR9BRA57N9_ejwDdQvz7B1tVxSEE,9367
|
|
797
798
|
datahub/metadata/schemas/DatasetDeprecation.avsc,sha256=ucXxaDcAUib9_y0k5qOINMn5VK2X3trHK2dcpNcsR2Q,1256
|
|
798
|
-
datahub/metadata/schemas/DatasetKey.avsc,sha256=
|
|
799
|
+
datahub/metadata/schemas/DatasetKey.avsc,sha256=51gyLo-nOcAGchs9pGAfv2vCpWR2y0hN1Xr4A26ms94,3496
|
|
799
800
|
datahub/metadata/schemas/DatasetProfile.avsc,sha256=3ZCU9JD6l2razACp0AY6LLMgnkMTj6D_5Xk9np6WWRM,9965
|
|
800
801
|
datahub/metadata/schemas/DatasetProperties.avsc,sha256=DFJn75feqaoQk84zin_o_lqsFFhqkwya5LGC5LLJXbU,4209
|
|
801
802
|
datahub/metadata/schemas/DatasetUpstreamLineage.avsc,sha256=PjAWPbsqwH7FjX2kFDy0dE6ENYOwRynH9vJerWisr2A,5365
|
|
@@ -858,6 +859,7 @@ datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=vNeAoFSIH1jEDa2rEyTStUj
|
|
|
858
859
|
datahub/metadata/schemas/IntendedUse.avsc,sha256=IKZSWdvc0uAyyT-FtdQOGbMC-P7RS9cO0vOVKWT6fbw,1361
|
|
859
860
|
datahub/metadata/schemas/InviteToken.avsc,sha256=8k_9MxHu9GVf7gvS0SlnQu7tJfpbXsRFdz6lQrFKPNc,737
|
|
860
861
|
datahub/metadata/schemas/InviteTokenKey.avsc,sha256=MuQUlQaeVjaBkjSshB9gsx5Fm0civYgWD8UhCiRLdOQ,434
|
|
862
|
+
datahub/metadata/schemas/LogicalParent.avsc,sha256=U7PlYOsE3klr4ksbE_VEKKR2uQZ1KwXIhMXmipWcZa0,5159
|
|
861
863
|
datahub/metadata/schemas/MLFeatureKey.avsc,sha256=1XFF8P2T3_4mX2oHGY74UJx1LFr6WtTv2jTG9ApjQoA,1150
|
|
862
864
|
datahub/metadata/schemas/MLFeatureProperties.avsc,sha256=HpF7VcnH2FvDsqy2g5AMJPqU7upkTVADW1ps-9bjINo,6893
|
|
863
865
|
datahub/metadata/schemas/MLFeatureTableKey.avsc,sha256=hVxNEqsx4GgG11GVryn9ms16OyEzsYdwM5RuAHy_454,1424
|
|
@@ -896,14 +898,14 @@ datahub/metadata/schemas/PostKey.avsc,sha256=MEA-S-RonGrsrpwYEejR6CXRka2DcqP7S7q
|
|
|
896
898
|
datahub/metadata/schemas/QuantitativeAnalyses.avsc,sha256=XrGYVoGUh7ZRjAbMORSiqF5Zl4a0IbYDPiQuSJhABZM,960
|
|
897
899
|
datahub/metadata/schemas/QueryKey.avsc,sha256=VI4oIHvAO7f0lN_7V3QVuBfHcPz31c57XtW6IrlDfxc,518
|
|
898
900
|
datahub/metadata/schemas/QueryProperties.avsc,sha256=26Q3zzuzJbUCUG7IJ3q_OEdcNbyzloZzDJWPbv3GQAk,5589
|
|
899
|
-
datahub/metadata/schemas/QuerySubjects.avsc,sha256=
|
|
901
|
+
datahub/metadata/schemas/QuerySubjects.avsc,sha256=PDM6Ek1AkjwLGDk17Rjg7KVtE2tbgFpc1XTGkhoG4MA,1437
|
|
900
902
|
datahub/metadata/schemas/QueryUsageStatistics.avsc,sha256=z1gfAnXdBoPEeERi5RESjrdBuS6AcIdqdN5JqWOSuNo,6192
|
|
901
903
|
datahub/metadata/schemas/RoleKey.avsc,sha256=Uas5jFViSHXhFqq8D4P6-UXqywOth3coztjQ5wA7wL0,449
|
|
902
904
|
datahub/metadata/schemas/RoleMembership.avsc,sha256=Al3LXKRowCiHhgTfwr3a-piID3Ld5kN-6-e9edTZ0uU,570
|
|
903
905
|
datahub/metadata/schemas/RoleProperties.avsc,sha256=tDw-WF1uBGIcrk38nOnXs3FCF_YjBhScarJbreQvwjE,3037
|
|
904
906
|
datahub/metadata/schemas/SchemaFieldAliases.avsc,sha256=El_cxn0KUhMf2LGfMPzcZ6Xtths2wQOaF9fnM1KQmxQ,560
|
|
905
907
|
datahub/metadata/schemas/SchemaFieldInfo.avsc,sha256=Gf9EGqrEf10554hd4Eut7T8ZdOR-9OHgDXVRhFo311o,800
|
|
906
|
-
datahub/metadata/schemas/SchemaFieldKey.avsc,sha256=
|
|
908
|
+
datahub/metadata/schemas/SchemaFieldKey.avsc,sha256=SnMq7HW-ctC22VVsaCb0Xdsr7S0FbpYN_5kvrjBujkY,1008
|
|
907
909
|
datahub/metadata/schemas/SchemaMetadata.avsc,sha256=aOuIAYN_ISWQRhRSBNvM6wb3oQz6o_V-DnUMn4oOUPU,40994
|
|
908
910
|
datahub/metadata/schemas/Siblings.avsc,sha256=lqzlNGP279TwFyFo7nUKx59atY2fyNGLlqIIJulSAz8,930
|
|
909
911
|
datahub/metadata/schemas/SlackUserInfo.avsc,sha256=IY7InWaiDzJa3hJ9J4W3Eg8EUKuh0_gHG801FGQTB6o,3411
|
|
@@ -1091,8 +1093,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1091
1093
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1092
1094
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1093
1095
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1094
|
-
acryl_datahub-1.1.0.
|
|
1095
|
-
acryl_datahub-1.1.0.
|
|
1096
|
-
acryl_datahub-1.1.0.
|
|
1097
|
-
acryl_datahub-1.1.0.
|
|
1098
|
-
acryl_datahub-1.1.0.
|
|
1096
|
+
acryl_datahub-1.1.0.5rc13.dist-info/METADATA,sha256=K0_AWqbWSWduRqBlxLCjnGD-7BQM1JYYH8y7dwzM6yE,181896
|
|
1097
|
+
acryl_datahub-1.1.0.5rc13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1098
|
+
acryl_datahub-1.1.0.5rc13.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
|
|
1099
|
+
acryl_datahub-1.1.0.5rc13.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1100
|
+
acryl_datahub-1.1.0.5rc13.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
|
@@ -162,12 +162,10 @@ class QuickstartVersionMappingConfig(BaseModel):
|
|
|
162
162
|
# docker compose based resolved compose file. In those cases, we pick up the composefile from
|
|
163
163
|
# MINIMUM_SUPPORTED_VERSION which contains the compose file.
|
|
164
164
|
if _is_it_a_version(result.composefile_git_ref):
|
|
165
|
-
if (
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
# https://github.com/datahub-project/datahub/pull/7473
|
|
170
|
-
result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e" # TODO update after compose file is merged
|
|
165
|
+
if parse("v1.2.0") > parse(result.composefile_git_ref):
|
|
166
|
+
# The merge commit where profiles based resolved compose file was added.
|
|
167
|
+
# https://github.com/datahub-project/datahub/pull/13566
|
|
168
|
+
result.composefile_git_ref = "21726bc3341490f4182b904626c793091ac95edd"
|
|
171
169
|
|
|
172
170
|
return result
|
|
173
171
|
|
datahub/ingestion/api/source.py
CHANGED
|
@@ -230,9 +230,10 @@ class SourceReport(ExamplesReport):
|
|
|
230
230
|
context: Optional[str] = None,
|
|
231
231
|
title: Optional[LiteralString] = None,
|
|
232
232
|
exc: Optional[BaseException] = None,
|
|
233
|
+
log: bool = True,
|
|
233
234
|
) -> None:
|
|
234
235
|
self._structured_logs.report_log(
|
|
235
|
-
StructuredLogLevel.WARN, message, title, context, exc, log=
|
|
236
|
+
StructuredLogLevel.WARN, message, title, context, exc, log=log
|
|
236
237
|
)
|
|
237
238
|
|
|
238
239
|
def report_failure(
|
|
@@ -533,7 +533,7 @@ class ABSSource(StatefulIngestionSourceBase):
|
|
|
533
533
|
)
|
|
534
534
|
path_spec.sample_files = False
|
|
535
535
|
for obj in container_client.list_blobs(
|
|
536
|
-
|
|
536
|
+
name_starts_with=f"{prefix}", results_per_page=PAGE_SIZE
|
|
537
537
|
):
|
|
538
538
|
abs_path = self.create_abs_path(obj.name)
|
|
539
539
|
logger.debug(f"Path: {abs_path}")
|
|
@@ -61,13 +61,13 @@ class AzureConnectionConfig(ConfigModel):
|
|
|
61
61
|
def get_blob_service_client(self):
|
|
62
62
|
return BlobServiceClient(
|
|
63
63
|
account_url=f"https://{self.account_name}.blob.core.windows.net",
|
|
64
|
-
credential=
|
|
64
|
+
credential=self.get_credentials(),
|
|
65
65
|
)
|
|
66
66
|
|
|
67
67
|
def get_data_lake_service_client(self) -> DataLakeServiceClient:
|
|
68
68
|
return DataLakeServiceClient(
|
|
69
69
|
account_url=f"https://{self.account_name}.dfs.core.windows.net",
|
|
70
|
-
credential=
|
|
70
|
+
credential=self.get_credentials(),
|
|
71
71
|
)
|
|
72
72
|
|
|
73
73
|
def get_credentials(
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import re
|
|
1
2
|
from abc import ABC, abstractmethod
|
|
2
3
|
|
|
3
4
|
# Add imports for source customization
|
|
@@ -236,42 +237,76 @@ class ABSObjectStore(ObjectStoreInterface):
|
|
|
236
237
|
"""Implementation of ObjectStoreInterface for Azure Blob Storage."""
|
|
237
238
|
|
|
238
239
|
PREFIX = "abfss://"
|
|
240
|
+
HTTPS_REGEX = re.compile(r"(https?://[a-z0-9]{3,24}\.blob\.core\.windows\.net/)")
|
|
239
241
|
|
|
240
242
|
@classmethod
|
|
241
243
|
def is_uri(cls, uri: str) -> bool:
|
|
242
|
-
return uri.startswith(cls.PREFIX)
|
|
244
|
+
return uri.startswith(cls.PREFIX) or bool(cls.HTTPS_REGEX.match(uri))
|
|
243
245
|
|
|
244
246
|
@classmethod
|
|
245
247
|
def get_prefix(cls, uri: str) -> Optional[str]:
|
|
246
248
|
if uri.startswith(cls.PREFIX):
|
|
247
249
|
return cls.PREFIX
|
|
250
|
+
|
|
251
|
+
# Check for HTTPS format
|
|
252
|
+
match = cls.HTTPS_REGEX.match(uri)
|
|
253
|
+
if match:
|
|
254
|
+
return match.group(1)
|
|
255
|
+
|
|
248
256
|
return None
|
|
249
257
|
|
|
250
258
|
@classmethod
|
|
251
259
|
def strip_prefix(cls, uri: str) -> str:
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
260
|
+
if uri.startswith(cls.PREFIX):
|
|
261
|
+
return uri[len(cls.PREFIX) :]
|
|
262
|
+
|
|
263
|
+
# Handle HTTPS format
|
|
264
|
+
match = cls.HTTPS_REGEX.match(uri)
|
|
265
|
+
if match:
|
|
266
|
+
return uri[len(match.group(1)) :]
|
|
267
|
+
|
|
268
|
+
raise ValueError(
|
|
269
|
+
f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
|
|
270
|
+
)
|
|
256
271
|
|
|
257
272
|
@classmethod
|
|
258
273
|
def get_bucket_name(cls, uri: str) -> str:
|
|
259
274
|
if not cls.is_uri(uri):
|
|
260
|
-
raise ValueError(
|
|
261
|
-
|
|
275
|
+
raise ValueError(
|
|
276
|
+
f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
if uri.startswith(cls.PREFIX):
|
|
280
|
+
# abfss://container@account.dfs.core.windows.net/path
|
|
281
|
+
return cls.strip_prefix(uri).split("@")[0]
|
|
282
|
+
else:
|
|
283
|
+
# https://account.blob.core.windows.net/container/path
|
|
284
|
+
return cls.strip_prefix(uri).split("/")[0]
|
|
262
285
|
|
|
263
286
|
@classmethod
|
|
264
287
|
def get_object_key(cls, uri: str) -> str:
|
|
265
288
|
if not cls.is_uri(uri):
|
|
266
|
-
raise ValueError(
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
289
|
+
raise ValueError(
|
|
290
|
+
f"Not an ABS URI. Must start with prefix: {cls.PREFIX} or match Azure Blob Storage HTTPS pattern"
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
if uri.startswith(cls.PREFIX):
|
|
294
|
+
# abfss://container@account.dfs.core.windows.net/path
|
|
295
|
+
parts = cls.strip_prefix(uri).split("@", 1)
|
|
296
|
+
if len(parts) < 2:
|
|
297
|
+
return ""
|
|
298
|
+
account_path = parts[1]
|
|
299
|
+
path_parts = account_path.split("/", 1)
|
|
300
|
+
if len(path_parts) < 2:
|
|
301
|
+
return ""
|
|
302
|
+
return path_parts[1]
|
|
303
|
+
else:
|
|
304
|
+
# https://account.blob.core.windows.net/container/path
|
|
305
|
+
stripped = cls.strip_prefix(uri)
|
|
306
|
+
parts = stripped.split("/", 1)
|
|
307
|
+
if len(parts) < 2:
|
|
308
|
+
return ""
|
|
309
|
+
return parts[1]
|
|
275
310
|
|
|
276
311
|
|
|
277
312
|
# Registry of all object store implementations
|
|
@@ -331,6 +366,12 @@ def get_object_store_bucket_name(uri: str) -> str:
|
|
|
331
366
|
return uri[prefix_length:].split("/")[0]
|
|
332
367
|
elif uri.startswith(ABSObjectStore.PREFIX):
|
|
333
368
|
return uri[len(ABSObjectStore.PREFIX) :].split("@")[0]
|
|
369
|
+
elif ABSObjectStore.HTTPS_REGEX.match(uri):
|
|
370
|
+
# Handle HTTPS Azure Blob Storage URLs
|
|
371
|
+
match = ABSObjectStore.HTTPS_REGEX.match(uri)
|
|
372
|
+
if match:
|
|
373
|
+
stripped = uri[len(match.group(1)) :]
|
|
374
|
+
return stripped.split("/")[0]
|
|
334
375
|
|
|
335
376
|
raise ValueError(f"Unsupported URI format: {uri}")
|
|
336
377
|
|
|
@@ -470,18 +511,25 @@ class ObjectStoreSourceAdapter:
|
|
|
470
511
|
if not ABSObjectStore.is_uri(table_data.table_path):
|
|
471
512
|
return None
|
|
472
513
|
|
|
473
|
-
# Parse the ABS URI
|
|
474
514
|
try:
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
515
|
+
if table_data.table_path.startswith("abfss://"):
|
|
516
|
+
# URI format: abfss://container@account.dfs.core.windows.net/path
|
|
517
|
+
path_without_prefix = ABSObjectStore.strip_prefix(table_data.table_path)
|
|
518
|
+
parts = path_without_prefix.split("@", 1)
|
|
519
|
+
if len(parts) < 2:
|
|
520
|
+
return None
|
|
521
|
+
|
|
522
|
+
container_name = parts[0]
|
|
523
|
+
account_parts = parts[1].split("/", 1)
|
|
524
|
+
account_domain = account_parts[0]
|
|
525
|
+
account_name = account_domain.split(".")[0]
|
|
526
|
+
else:
|
|
527
|
+
# Handle HTTPS format: https://account.blob.core.windows.net/container/path
|
|
528
|
+
container_name = ABSObjectStore.get_bucket_name(table_data.table_path)
|
|
529
|
+
if "blob.core.windows.net" in table_data.table_path:
|
|
530
|
+
account_name = table_data.table_path.split("//")[1].split(".")[0]
|
|
531
|
+
else:
|
|
532
|
+
return None
|
|
485
533
|
|
|
486
534
|
# Construct Azure portal URL
|
|
487
535
|
return f"https://portal.azure.com/#blade/Microsoft_Azure_Storage/ContainerMenuBlade/overview/storageAccountId/{account_name}/containerName/{container_name}"
|
|
@@ -26,6 +26,7 @@ from datahub.ingestion.source.dbt.dbt_common import (
|
|
|
26
26
|
DBTCommonConfig,
|
|
27
27
|
DBTNode,
|
|
28
28
|
DBTSourceBase,
|
|
29
|
+
DBTSourceReport,
|
|
29
30
|
)
|
|
30
31
|
from datahub.ingestion.source.dbt.dbt_tests import DBTTest, DBTTestResult
|
|
31
32
|
|
|
@@ -266,6 +267,7 @@ query DatahubMetadataQuery_{type}($jobId: BigInt!, $runId: BigInt) {{
|
|
|
266
267
|
@capability(SourceCapability.TEST_CONNECTION, "Enabled by default")
|
|
267
268
|
class DBTCloudSource(DBTSourceBase, TestableSource):
|
|
268
269
|
config: DBTCloudConfig
|
|
270
|
+
report: DBTSourceReport # nothing cloud-specific in the report
|
|
269
271
|
|
|
270
272
|
@classmethod
|
|
271
273
|
def create(cls, config_dict, ctx):
|
|
@@ -404,8 +406,11 @@ class DBTCloudSource(DBTSourceBase, TestableSource):
|
|
|
404
406
|
if node["resourceType"] in {"model", "seed", "snapshot"}:
|
|
405
407
|
status = node["status"]
|
|
406
408
|
if status is None and materialization != "ephemeral":
|
|
407
|
-
self.report.
|
|
408
|
-
|
|
409
|
+
self.report.warning(
|
|
410
|
+
title="Schema information may be incomplete",
|
|
411
|
+
message="Some nodes are missing the `status` field, which dbt uses to track the status of the node in the target database.",
|
|
412
|
+
context=key,
|
|
413
|
+
log=False,
|
|
409
414
|
)
|
|
410
415
|
|
|
411
416
|
# The code fields are new in dbt 1.3, and replace the sql ones.
|
|
@@ -1336,6 +1336,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
|
|
1336
1336
|
self.config.tag_prefix,
|
|
1337
1337
|
"SOURCE_CONTROL",
|
|
1338
1338
|
self.config.strip_user_ids_from_email,
|
|
1339
|
+
match_nested_props=True,
|
|
1339
1340
|
)
|
|
1340
1341
|
|
|
1341
1342
|
action_processor_tag = OperationProcessor(
|
|
@@ -1707,6 +1708,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
|
|
1707
1708
|
self.config.tag_prefix,
|
|
1708
1709
|
"SOURCE_CONTROL",
|
|
1709
1710
|
self.config.strip_user_ids_from_email,
|
|
1711
|
+
match_nested_props=True,
|
|
1710
1712
|
)
|
|
1711
1713
|
|
|
1712
1714
|
canonical_schema: List[SchemaField] = []
|
|
@@ -70,10 +70,19 @@ class LineageConfigGen1(ConfigModel):
|
|
|
70
70
|
Table naming convention: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
|
|
71
71
|
"""
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
enabled: bool = Field(
|
|
74
74
|
default=False,
|
|
75
|
+
description="Whether this source is enabled",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
emit_lineage: bool = Field(
|
|
79
|
+
default=True,
|
|
75
80
|
description="Whether to emit lineage data for testing purposes. When False, no lineage data is generated regardless of other settings.",
|
|
76
81
|
)
|
|
82
|
+
emit_usage: bool = Field(
|
|
83
|
+
default=True,
|
|
84
|
+
description="Whether to emit usage data for testing purposes. When False, no usage data is generated regardless of other settings.",
|
|
85
|
+
)
|
|
77
86
|
|
|
78
87
|
lineage_fan_out: int = Field(
|
|
79
88
|
default=3,
|
|
@@ -106,6 +115,18 @@ class DataHubMockDataConfig(ConfigModel):
|
|
|
106
115
|
default=True,
|
|
107
116
|
description="Whether this source is enabled",
|
|
108
117
|
)
|
|
118
|
+
throw_uncaught_exceptions: bool = Field(
|
|
119
|
+
default=False,
|
|
120
|
+
description="Whether to throw an uncaught exception for testing",
|
|
121
|
+
)
|
|
122
|
+
num_errors: int = Field(
|
|
123
|
+
default=0,
|
|
124
|
+
description="Number of errors to add in report for testing",
|
|
125
|
+
)
|
|
126
|
+
num_warnings: int = Field(
|
|
127
|
+
default=0,
|
|
128
|
+
description="Number of warnings to add in report for testing",
|
|
129
|
+
)
|
|
109
130
|
|
|
110
131
|
gen_1: LineageConfigGen1 = Field(
|
|
111
132
|
default_factory=LineageConfigGen1,
|
|
@@ -128,9 +149,28 @@ class DataHubMockDataSource(Source):
|
|
|
128
149
|
self.report = DataHubMockDataReport()
|
|
129
150
|
|
|
130
151
|
def get_workunits(self) -> Iterable[MetadataWorkUnit]:
|
|
152
|
+
if self.config.throw_uncaught_exceptions:
|
|
153
|
+
raise Exception("This is a test exception")
|
|
154
|
+
|
|
155
|
+
if self.config.num_errors > 0:
|
|
156
|
+
for i in range(self.config.num_errors):
|
|
157
|
+
self.report.failure(
|
|
158
|
+
message="This is test error message",
|
|
159
|
+
title="Test Error",
|
|
160
|
+
context=f"This is test error {i}",
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
if self.config.num_warnings > 0:
|
|
164
|
+
for i in range(self.config.num_warnings):
|
|
165
|
+
self.report.warning(
|
|
166
|
+
message="This is test warning",
|
|
167
|
+
title="Test Warning",
|
|
168
|
+
context=f"This is test warning {i}",
|
|
169
|
+
)
|
|
170
|
+
|
|
131
171
|
# We don't want any implicit aspects to be produced
|
|
132
172
|
# so we are not using get_workunits_internal
|
|
133
|
-
if self.config.gen_1.
|
|
173
|
+
if self.config.gen_1.enabled:
|
|
134
174
|
for wu in self._data_gen_1():
|
|
135
175
|
if self.report.first_urn_seen is None:
|
|
136
176
|
self.report.first_urn_seen = wu.get_urn()
|
|
@@ -284,17 +324,19 @@ class DataHubMockDataSource(Source):
|
|
|
284
324
|
|
|
285
325
|
yield self._get_profile_aspect(table_name)
|
|
286
326
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
327
|
+
if self.config.gen_1.emit_usage:
|
|
328
|
+
yield self._get_usage_aspect(table_name)
|
|
329
|
+
|
|
330
|
+
if self.config.gen_1.emit_lineage:
|
|
331
|
+
yield from self._generate_lineage_for_table(
|
|
332
|
+
table_name=table_name,
|
|
333
|
+
table_level=i,
|
|
334
|
+
table_index=j,
|
|
335
|
+
hops=hops,
|
|
336
|
+
fan_out=fan_out,
|
|
337
|
+
fan_out_after_first=fan_out_after_first,
|
|
338
|
+
tables_at_levels=tables_at_levels,
|
|
339
|
+
)
|
|
298
340
|
|
|
299
341
|
def _generate_lineage_for_table(
|
|
300
342
|
self,
|
|
@@ -236,7 +236,7 @@ class SnowflakeV2Config(
|
|
|
236
236
|
)
|
|
237
237
|
|
|
238
238
|
use_queries_v2: bool = Field(
|
|
239
|
-
default=
|
|
239
|
+
default=True,
|
|
240
240
|
description="If enabled, uses the new queries extractor to extract queries from snowflake.",
|
|
241
241
|
)
|
|
242
242
|
include_queries: bool = Field(
|
|
@@ -361,6 +361,20 @@ class SnowflakeV2Config(
|
|
|
361
361
|
"Only applicable if `use_queries_v2` is enabled.",
|
|
362
362
|
)
|
|
363
363
|
|
|
364
|
+
push_down_database_pattern_access_history: bool = Field(
|
|
365
|
+
default=False,
|
|
366
|
+
description="If enabled, pushes down database pattern filtering to the access_history table for improved performance. "
|
|
367
|
+
"This filters on the accessed objects in access_history.",
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
additional_database_names_allowlist: List[str] = Field(
|
|
371
|
+
default=[],
|
|
372
|
+
description="Additional database names (no pattern matching) to be included in the access_history filter. "
|
|
373
|
+
"Only applies if push_down_database_pattern_access_history=True. "
|
|
374
|
+
"These databases will be included in the filter being pushed down regardless of database_pattern settings."
|
|
375
|
+
"This may be required in the case of _eg_ temporary tables being created in a different database than the ones in the database_name patterns.",
|
|
376
|
+
)
|
|
377
|
+
|
|
364
378
|
@validator("convert_urns_to_lowercase")
|
|
365
379
|
def validate_convert_urns_to_lowercase(cls, v):
|
|
366
380
|
if not v:
|