acryl-datahub 1.1.0.5rc5__py3-none-any.whl → 1.1.0.5rc7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/METADATA +2603 -2603
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/RECORD +43 -39
- datahub/_version.py +1 -1
- datahub/ingestion/api/report.py +183 -35
- datahub/ingestion/autogenerated/capability_summary.json +3366 -0
- datahub/ingestion/autogenerated/lineage.json +401 -0
- datahub/ingestion/autogenerated/lineage_helper.py +30 -128
- datahub/ingestion/run/pipeline.py +4 -1
- datahub/ingestion/source/bigquery_v2/bigquery.py +23 -22
- datahub/ingestion/source/bigquery_v2/queries.py +2 -2
- datahub/ingestion/source/cassandra/cassandra_profiling.py +6 -5
- datahub/ingestion/source/common/subtypes.py +1 -1
- datahub/ingestion/source/data_lake_common/object_store.py +40 -0
- datahub/ingestion/source/data_lake_common/path_spec.py +10 -21
- datahub/ingestion/source/dremio/dremio_source.py +6 -3
- datahub/ingestion/source/gcs/gcs_source.py +4 -1
- datahub/ingestion/source/ge_data_profiler.py +28 -20
- datahub/ingestion/source/kafka_connect/source_connectors.py +59 -4
- datahub/ingestion/source/mock_data/datahub_mock_data.py +45 -0
- datahub/ingestion/source/redshift/usage.py +4 -3
- datahub/ingestion/source/s3/report.py +4 -2
- datahub/ingestion/source/s3/source.py +367 -115
- datahub/ingestion/source/snowflake/snowflake_queries.py +47 -3
- datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
- datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
- datahub/ingestion/source/sql/athena.py +95 -10
- datahub/ingestion/source/sql/athena_properties_extractor.py +777 -0
- datahub/ingestion/source/unity/proxy.py +4 -3
- datahub/ingestion/source/unity/source.py +10 -8
- datahub/integrations/assertion/snowflake/compiler.py +4 -3
- datahub/metadata/_internal_schema_classes.py +85 -4
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +2 -0
- datahub/metadata/schema.avsc +54 -1
- datahub/metadata/schemas/CorpUserSettings.avsc +17 -1
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +37 -0
- datahub/sdk/lineage_client.py +2 -0
- datahub/sql_parsing/sql_parsing_aggregator.py +3 -3
- datahub/sql_parsing/sqlglot_lineage.py +2 -0
- datahub/utilities/sqlalchemy_query_combiner.py +5 -2
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.1.0.5rc5.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.1.0.
|
|
1
|
+
acryl_datahub-1.1.0.5rc7.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=sUspZGha0FX624vViKEbY5GTEpcVIfY_B0Gq4roHDZ0,323
|
|
5
5
|
datahub/entrypoints.py,sha256=H-YFTvxTJOgpWsFBVlxyb1opjkq-hjTzNmjy5Fq3RHg,8992
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -148,7 +148,7 @@ datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPs
|
|
|
148
148
|
datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
|
|
149
149
|
datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
|
|
150
150
|
datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
|
|
151
|
-
datahub/ingestion/api/report.py,sha256=
|
|
151
|
+
datahub/ingestion/api/report.py,sha256=SRZ1DNyHUV08uxA3qGzWOX-lVfSSwh4ln6NOSwMYhIA,14807
|
|
152
152
|
datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
|
|
153
153
|
datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
|
|
154
154
|
datahub/ingestion/api/source.py,sha256=WcpmuWRua87_UbaSY8GtkLkC0Gql0pM_i5FJN5ALh88,19669
|
|
@@ -159,7 +159,9 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
|
159
159
|
datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
|
|
160
160
|
datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=fMjPnyWEofIZV52E2AFYU3IgBJwyZvbygXxCJyEtcWI,4442
|
|
161
161
|
datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
datahub/ingestion/autogenerated/
|
|
162
|
+
datahub/ingestion/autogenerated/capability_summary.json,sha256=CiZ0OZlU40jaItCrOYwdUfnb2JVZSb-KYBr2-MiMfpc,104572
|
|
163
|
+
datahub/ingestion/autogenerated/lineage.json,sha256=wjaXxrzpo3M9ppc80TKIgCTuCg0jpb-D9MhBsHXaMOY,10004
|
|
164
|
+
datahub/ingestion/autogenerated/lineage_helper.py,sha256=MJrHGuDvM_9hrFfRt0dkzE58j1xIsvgW6UIQBEKqdYE,2701
|
|
163
165
|
datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
164
166
|
datahub/ingestion/extractor/extractor_registry.py,sha256=f7CLfW3pr29QZkXSHbp7HjUrsdw7ejQJmot-tiSPcqc,342
|
|
165
167
|
datahub/ingestion/extractor/json_ref_patch.py,sha256=4g3ZWHn7rwS74jUvSXJiGpi-UKHhiSYKKgBeU4E5ukE,1448
|
|
@@ -191,7 +193,7 @@ datahub/ingestion/reporting/file_reporter.py,sha256=tiWukmMxHrTQI3rOAumsq6lRlw8T
|
|
|
191
193
|
datahub/ingestion/reporting/reporting_provider_registry.py,sha256=jTYSh3T4sensjnHQfPLiIcbA2dG8w0px9ghChAJjGdU,310
|
|
192
194
|
datahub/ingestion/run/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
193
195
|
datahub/ingestion/run/connection.py,sha256=mngNzr5aRLUDa5Izqxa0xkdDEqEqcDuacWSKIlkdvPc,1483
|
|
194
|
-
datahub/ingestion/run/pipeline.py,sha256=
|
|
196
|
+
datahub/ingestion/run/pipeline.py,sha256=FGRvCo9aRIMtDa4a3JHy1u2xi70p9TQuiiC17yXZVSc,30287
|
|
195
197
|
datahub/ingestion/run/pipeline_config.py,sha256=joG1j9OlwJhb8zqv4TY6_FSzOaKOx6xsBu255A5lP8g,4101
|
|
196
198
|
datahub/ingestion/run/sink_callback.py,sha256=xZAzaKkR0dcQP838pMJWsA52oaQXV5BiqXkpxEpJ_9U,2856
|
|
197
199
|
datahub/ingestion/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -209,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
|
|
|
209
211
|
datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
|
|
210
212
|
datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
|
|
211
213
|
datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
|
|
212
|
-
datahub/ingestion/source/ge_data_profiler.py,sha256=
|
|
214
|
+
datahub/ingestion/source/ge_data_profiler.py,sha256=TnAupfORYFzcgP7akQEqeuvy1qO0sxMetb0B75f95jM,67980
|
|
213
215
|
datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
|
|
214
216
|
datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
|
|
215
217
|
datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
|
|
@@ -254,7 +256,7 @@ datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0
|
|
|
254
256
|
datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
|
|
255
257
|
datahub/ingestion/source/azure/azure_common.py,sha256=Zl0pPuE6L3QcM5B1P0LsPthZmD0h7fUUS0kg2okl6IY,4053
|
|
256
258
|
datahub/ingestion/source/bigquery_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
257
|
-
datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=
|
|
259
|
+
datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=QNG_3IU6xFyGz7Xq2dHyPS0C-LCuJyMAFpqPrMLRBsE,15100
|
|
258
260
|
datahub/ingestion/source/bigquery_v2/bigquery_audit.py,sha256=kEwWhq3ch6WT4q4hcX8-fvQh28KgrNfspFwIytO3vQA,25103
|
|
259
261
|
datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py,sha256=LuGJ6LgPViLIfDQfylxlQ3CA7fZYM5MDt8M-7sfzm84,5096
|
|
260
262
|
datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=bG1soRawCLhJw_15L2fJmFfj1kntTthV6ng4LZOnwko,21916
|
|
@@ -270,25 +272,25 @@ datahub/ingestion/source/bigquery_v2/bigquery_test_connection.py,sha256=cATxwi5I
|
|
|
270
272
|
datahub/ingestion/source/bigquery_v2/common.py,sha256=IinOy-RO4UZGxSf5scaN02672BzZuNsjJZ56axti6iI,4016
|
|
271
273
|
datahub/ingestion/source/bigquery_v2/lineage.py,sha256=jju14mJbAUMA_K3j2yq-TdZV202cjd5rBAsDPJGEVno,44900
|
|
272
274
|
datahub/ingestion/source/bigquery_v2/profiler.py,sha256=oLf5jMjJf-ShNny9Dll2tCsOoPMF1DxAh7e7etpeLq4,10821
|
|
273
|
-
datahub/ingestion/source/bigquery_v2/queries.py,sha256=
|
|
275
|
+
datahub/ingestion/source/bigquery_v2/queries.py,sha256=gDvvgajptmNn5AiBglmDhGAC9LBh8fzw56_d8ewLbxA,20222
|
|
274
276
|
datahub/ingestion/source/bigquery_v2/queries_extractor.py,sha256=_5cAXVU8b8T_nAPDsvN2JRd2dmM1t1J1mRylfKiPen4,19530
|
|
275
277
|
datahub/ingestion/source/bigquery_v2/usage.py,sha256=A9c-ofclaRk0NSnc4IRaqJYqMPv6ecCld_TPy3V2qFs,40748
|
|
276
278
|
datahub/ingestion/source/cassandra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
277
279
|
datahub/ingestion/source/cassandra/cassandra.py,sha256=k2EKpeUUZ0O_RwQ6NXM15vnTsY3OgST4e5LG8Eg6Coo,14534
|
|
278
280
|
datahub/ingestion/source/cassandra/cassandra_api.py,sha256=b7MApc3_tEfHoj-6ub6snkcv_DweL1wi_TGJjAA1-yU,13516
|
|
279
281
|
datahub/ingestion/source/cassandra/cassandra_config.py,sha256=Ga9915cDZukR5-u2tMNx5Jkf8eza2oAE5YS_sQIVEVQ,4222
|
|
280
|
-
datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=
|
|
282
|
+
datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=FdgPK_8s8otTOJDqNM4rpF6Mn4lFWbnjTaKEChzn2iE,11011
|
|
281
283
|
datahub/ingestion/source/cassandra/cassandra_utils.py,sha256=j-LidYkaCTmGnpUVNLsax_c3z32PsQbsbHeYojygd1s,5105
|
|
282
284
|
datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
283
285
|
datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
|
|
284
286
|
datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
|
|
285
287
|
datahub/ingestion/source/common/gcp_credentials_config.py,sha256=_NapGkAqZMbXNClLlmOfANS7U6rChhdthRX9s9iUv9k,2411
|
|
286
|
-
datahub/ingestion/source/common/subtypes.py,sha256
|
|
288
|
+
datahub/ingestion/source/common/subtypes.py,sha256=-z-QseXQC-Pt8pbbw4CsMRsqIcNzZ-mz-9iQSXyU65s,4493
|
|
287
289
|
datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
288
290
|
datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
|
|
289
291
|
datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGDTUOol9Oi3jZs8Pb04PIsHdC2WPP0dkg,6576
|
|
290
|
-
datahub/ingestion/source/data_lake_common/object_store.py,sha256=
|
|
291
|
-
datahub/ingestion/source/data_lake_common/path_spec.py,sha256=
|
|
292
|
+
datahub/ingestion/source/data_lake_common/object_store.py,sha256=NkbCXq56sdGkyBAxcrpSVE6GySLrh9E2jYZU6lYPupM,21644
|
|
293
|
+
datahub/ingestion/source/data_lake_common/path_spec.py,sha256=ekJAr4-PE2RhzQnmKb2xcSs_YncC1Dz95-UrXI67Vos,23584
|
|
292
294
|
datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
293
295
|
datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
|
|
294
296
|
datahub/ingestion/source/datahub/datahub_api_reader.py,sha256=hlKADVEPoTFiRGKqRsMF5mL4fSu_IrIW8Nx7LpEzvkM,2134
|
|
@@ -317,7 +319,7 @@ datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py,sha256=MQk8BAHL
|
|
|
317
319
|
datahub/ingestion/source/dremio/dremio_entities.py,sha256=1gZrNqTp3Pm6vqGDQaWt3HkxEuHKxpGYQ4geVoFvxWI,15147
|
|
318
320
|
datahub/ingestion/source/dremio/dremio_profiling.py,sha256=TAcnpo8ZRKhLDHnQSJzJg3YdwTSyEa73LUAzENs7wG4,12287
|
|
319
321
|
datahub/ingestion/source/dremio/dremio_reporting.py,sha256=YRKM6PvoJYHLBXmOGwkgou_8x8_oA2xaqTWWoVuwFMY,2247
|
|
320
|
-
datahub/ingestion/source/dremio/dremio_source.py,sha256=
|
|
322
|
+
datahub/ingestion/source/dremio/dremio_source.py,sha256=ODIUhDr6Nx5ziV7Buo-ayT40WxkSKNhFhOlM6fmw-xM,25236
|
|
321
323
|
datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=wA1hqKk9cKMJDyEdZRQcDDLZPGYwuNqrvleUHTkWgrQ,10508
|
|
322
324
|
datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
323
325
|
datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
|
|
@@ -334,7 +336,7 @@ datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=mUWcMt-_FL1SYGIgI4lGZD
|
|
|
334
336
|
datahub/ingestion/source/gc/execution_request_cleanup.py,sha256=y-9ZIs_DZPUzYH1CI6HmaAZg3olNNA7MjT8HrCqAI0k,11159
|
|
335
337
|
datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py,sha256=4-qQR_2HGIYU8kC2hRIsJyKKMb9lKq4B6paJm_abUk4,12628
|
|
336
338
|
datahub/ingestion/source/gcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
337
|
-
datahub/ingestion/source/gcs/gcs_source.py,sha256=
|
|
339
|
+
datahub/ingestion/source/gcs/gcs_source.py,sha256=mrecg2ewT2ATrUuEYg3cMLPXXAWwc2WJzJFmAHNWuTc,7063
|
|
338
340
|
datahub/ingestion/source/gcs/gcs_utils.py,sha256=Kd2usZYIMFeSuE6_tJ4OoHGOdvG8mWaScFuAcIkC6P0,1789
|
|
339
341
|
datahub/ingestion/source/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
340
342
|
datahub/ingestion/source/git/git_import.py,sha256=5CT6vMDb0MDctCtShnxb3JVihULtvkYGr9judHJFsOk,4143
|
|
@@ -362,7 +364,7 @@ datahub/ingestion/source/kafka_connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
|
|
|
362
364
|
datahub/ingestion/source/kafka_connect/common.py,sha256=lH64n1v_rJamWGfidBeuQJj8W1_IvOBpXQLR2YZaEvQ,7057
|
|
363
365
|
datahub/ingestion/source/kafka_connect/kafka_connect.py,sha256=AVAgBvgH7kM9I2ke3mwr8CfIL1J2SdVHH_86rnCFwrM,17727
|
|
364
366
|
datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfICTXIsv3ZfzlrMA4VlZXKQTcFCyIAU,17220
|
|
365
|
-
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=
|
|
367
|
+
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
|
|
366
368
|
datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
367
369
|
datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
|
|
368
370
|
datahub/ingestion/source/looker/looker_common.py,sha256=wwCRHyd0F799CEBboryQ4TZV057krnY2nMJOkUWc268,64765
|
|
@@ -389,7 +391,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
|
|
|
389
391
|
datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
|
|
390
392
|
datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
|
|
391
393
|
datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
392
|
-
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=
|
|
394
|
+
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=PLQU1WnxDljZJGifoh6CqIQdBZrQAeBxtV-9NLdxM8o,15847
|
|
393
395
|
datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
|
|
394
396
|
datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
|
|
395
397
|
datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -438,13 +440,13 @@ datahub/ingestion/source/redshift/redshift.py,sha256=p6rOOCjxNnPpTn-vFjgISMMjtUT
|
|
|
438
440
|
datahub/ingestion/source/redshift/redshift_data_reader.py,sha256=zc69jwXHdF-w8J4Hq-ZQ6BjHQ75Ij2iNDMpoRJlcmlU,1724
|
|
439
441
|
datahub/ingestion/source/redshift/redshift_schema.py,sha256=7F-l_omOuKMuGE_rBWXVPG_GWXFKnCMzC4frNxZB9cs,24800
|
|
440
442
|
datahub/ingestion/source/redshift/report.py,sha256=O3QFozHlmMbH9b7KxbqhgTgr_0tCryj6FIzMiN6kRxw,3044
|
|
441
|
-
datahub/ingestion/source/redshift/usage.py,sha256=
|
|
443
|
+
datahub/ingestion/source/redshift/usage.py,sha256=Q7R-caJovLXv33uZepMGX5Cvm4DqQSLZdiL_s-p06wU,17473
|
|
442
444
|
datahub/ingestion/source/s3/__init__.py,sha256=HjqFPj11WtNFZM3kcVshlDb7kOsc19-l_3LM8PBjlJM,56
|
|
443
445
|
datahub/ingestion/source/s3/config.py,sha256=lElFXgEpKDT9SVoiXvtx98wV6Gp880qP4pLQaOGJGOo,7828
|
|
444
446
|
datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
|
|
445
447
|
datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
|
|
446
|
-
datahub/ingestion/source/s3/report.py,sha256=
|
|
447
|
-
datahub/ingestion/source/s3/source.py,sha256=
|
|
448
|
+
datahub/ingestion/source/s3/report.py,sha256=9Ej1UCChw963UpGw1-7asi5vFrOM232gfgG8bRdKPp0,667
|
|
449
|
+
datahub/ingestion/source/s3/source.py,sha256=DuJ5G0S2ukZFRL-aO8b4ihzhTlxvbzbj2_GyS8zLb7s,60968
|
|
448
450
|
datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
449
451
|
datahub/ingestion/source/sac/sac.py,sha256=0s_JxHGOhit3Wvgbg7qQi-Z9j9_TgBX_I1yOR3L6-rA,30243
|
|
450
452
|
datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
|
|
@@ -475,7 +477,7 @@ datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUH
|
|
|
475
477
|
datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
|
|
476
478
|
datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
|
|
477
479
|
datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
|
|
478
|
-
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=
|
|
480
|
+
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=PHDFks4rhSfqsPLcamMZJDDJic8988P3WNWglXIUN6M,33017
|
|
479
481
|
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=kqOxibplHyDhdioue8MeXBzeDS9d274-dspb-wyWMMI,38165
|
|
480
482
|
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
|
|
481
483
|
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
|
|
@@ -483,11 +485,13 @@ datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=_37-AQyI4uGt4f
|
|
|
483
485
|
datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
|
|
484
486
|
datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GKONtWCTtYvTdR209cRW9g66xyYE,5671
|
|
485
487
|
datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
|
|
486
|
-
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=
|
|
488
|
+
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
|
|
487
489
|
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=w-SftpjHSOf_6S2T2srHJyJta3MHS0usmn4Z9jgx4QE,13858
|
|
488
490
|
datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=eqOVRYmFiWQPAIDWUcOo5QZkG05xanlEbxz4aDTvSNM,34706
|
|
491
|
+
datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=XVEP8rGGIqds8f7RRoUHGajsT0myBpYS_u5aVdwggBU,5308
|
|
489
492
|
datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
490
|
-
datahub/ingestion/source/sql/athena.py,sha256=
|
|
493
|
+
datahub/ingestion/source/sql/athena.py,sha256=lQ_zniHtmtSFCnSmzM2dG7ND6zOZmhu6t-I4ZOeQKho,27652
|
|
494
|
+
datahub/ingestion/source/sql/athena_properties_extractor.py,sha256=n2SvqeUbNWxiWWdkDs8VYlUPlLwfZzZy9AIa-V4D7AY,28531
|
|
491
495
|
datahub/ingestion/source/sql/clickhouse.py,sha256=_uf-6fQYxI1-kMw0dNUvJf1f75XX0Qps4HMVlgzdpmo,25670
|
|
492
496
|
datahub/ingestion/source/sql/cockroachdb.py,sha256=WoOKCq7YjsuzSPm1SmKIYZ9CrvlSF8zWmP1fNHn4G3Q,1360
|
|
493
497
|
datahub/ingestion/source/sql/druid.py,sha256=_tzgTa5jhPUXk6WCmS7p10feCwJm6yUFcOgMZA-OcE8,2922
|
|
@@ -546,11 +550,11 @@ datahub/ingestion/source/unity/config.py,sha256=7QosoBthg9kirHfXev_vhefkobUxYnp1
|
|
|
546
550
|
datahub/ingestion/source/unity/connection_test.py,sha256=B143Wb28fS0V4GhygU9hzKqiArWBjsQO54IUCPf23dc,2586
|
|
547
551
|
datahub/ingestion/source/unity/ge_profiler.py,sha256=NBRHZceq-f95iUn7u0h7cgcd9nAc48Aa-lmp_BqE0As,8409
|
|
548
552
|
datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
|
|
549
|
-
datahub/ingestion/source/unity/proxy.py,sha256=
|
|
553
|
+
datahub/ingestion/source/unity/proxy.py,sha256=i15xEfjt7sbajbDIA9cbrxKWLByoL2drI8lnvWPceqc,22787
|
|
550
554
|
datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
|
|
551
555
|
datahub/ingestion/source/unity/proxy_types.py,sha256=qrvHiwPzl5cPX-KRvcIGGeJVdr0I8XUQmoAI6ErZ-v8,9371
|
|
552
556
|
datahub/ingestion/source/unity/report.py,sha256=XFT9oQfvEB4RkTvWGgFOoQuLPUN_AIoPXZ79xeDhGHQ,2831
|
|
553
|
-
datahub/ingestion/source/unity/source.py,sha256=
|
|
557
|
+
datahub/ingestion/source/unity/source.py,sha256=_cqxiQYZWyNXHpnpZ6EGO9WRfL6a_7fwLwv5cVZdto4,49826
|
|
554
558
|
datahub/ingestion/source/unity/tag_entities.py,sha256=iWl6nRAWSye1hoFDx_Xh4aT53PN0sGzlX7n1-oTVUv8,11568
|
|
555
559
|
datahub/ingestion/source/unity/usage.py,sha256=0wETBAaZvHI_EGgBlxX3bKsVHEAdnUV8_bKI_lbyWjY,11500
|
|
556
560
|
datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -598,7 +602,7 @@ datahub/integrations/assertion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
|
|
|
598
602
|
datahub/integrations/assertion/common.py,sha256=rUkcyJ87vedWqJPVGco0e6FPF07zj8DNOPW1knIvDg8,2297
|
|
599
603
|
datahub/integrations/assertion/registry.py,sha256=mmeYpQREFVrLWpLcA0qYZtbrMX4vAGcDoQ59wtMgEcg,307
|
|
600
604
|
datahub/integrations/assertion/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
601
|
-
datahub/integrations/assertion/snowflake/compiler.py,sha256=
|
|
605
|
+
datahub/integrations/assertion/snowflake/compiler.py,sha256=qL757BVcpR9MoUOHPKBmDoid_5OSNbCUbbt3BivEds8,9970
|
|
602
606
|
datahub/integrations/assertion/snowflake/dmf_generator.py,sha256=ti4EnSlTGdAWmBcxgUqqilCGhLVGutpfSatbENP94O8,746
|
|
603
607
|
datahub/integrations/assertion/snowflake/field_metric_sql_generator.py,sha256=V2kwDTgg3ChnxW2h5pITpUwtCiwL9Lmhg4p6T5iS9IE,6513
|
|
604
608
|
datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py,sha256=aVVOhYPZKVITVNfGEQO54_XjjRuMplsu-yeFLE7MhxA,8950
|
|
@@ -614,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
|
|
|
614
618
|
datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
|
|
615
619
|
datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
|
|
616
620
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
617
|
-
datahub/metadata/_internal_schema_classes.py,sha256=
|
|
618
|
-
datahub/metadata/schema.avsc,sha256=
|
|
621
|
+
datahub/metadata/_internal_schema_classes.py,sha256=h_FmxHQdH1tChgFYOGgULr6RMgiXN9JfFOoMC3TSoEg,1043586
|
|
622
|
+
datahub/metadata/schema.avsc,sha256=zrngtDblYg2sLuu-5t5IJxRq52TexvIIElfoU99wuZw,720852
|
|
619
623
|
datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
|
|
620
624
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
621
625
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -686,7 +690,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schema/__init__.py,sha256=7JpzLs6S_Ey
|
|
|
686
690
|
datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWeznycKnHBfPEGcCHXPEz83Iq9ypjNaoSfeQeDU9g,397
|
|
687
691
|
datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
|
|
688
692
|
datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
|
|
689
|
-
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=
|
|
693
|
+
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=hKpTDXj2YEBja4hk2fZx3G9yzJ7eCFxcGCbbN0NydTk,933
|
|
690
694
|
datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
|
|
691
695
|
datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
|
|
692
696
|
datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
|
|
@@ -727,7 +731,7 @@ datahub/metadata/schemas/CorpUserCredentials.avsc,sha256=S7FkV9K_DGxhb4GFYbM5_lP
|
|
|
727
731
|
datahub/metadata/schemas/CorpUserEditableInfo.avsc,sha256=6IrqWidbHP7mRryfVlWAQU0JS34THHTM8_aIKWqClUE,3843
|
|
728
732
|
datahub/metadata/schemas/CorpUserInfo.avsc,sha256=oObOza-5FLjZyCjj0FN4MNV1DodgTwJSV4APduAggjk,3955
|
|
729
733
|
datahub/metadata/schemas/CorpUserKey.avsc,sha256=01sbbdr8G-ZP1yc2UfY3gR-YN6b7AvDbCbNpZJ-02J4,1025
|
|
730
|
-
datahub/metadata/schemas/CorpUserSettings.avsc,sha256=
|
|
734
|
+
datahub/metadata/schemas/CorpUserSettings.avsc,sha256=ff-rvW3ihwo6s9usvStx0aXTmCNmcR1TjggDxhbCacc,6529
|
|
731
735
|
datahub/metadata/schemas/CorpUserStatus.avsc,sha256=yqojAXEQ9CjRhY58RPyTUxzmFbHSANGGaMMbqiYZZIE,2538
|
|
732
736
|
datahub/metadata/schemas/Cost.avsc,sha256=o4kYZSss2uEwJ6gCA9fhBUoyD5xUqcSxz78vkIXXzGQ,1494
|
|
733
737
|
datahub/metadata/schemas/DashboardInfo.avsc,sha256=li2lSV6R4V-nz6foOi-NYxt_8ShHWfoKRw6M2BG5530,12907
|
|
@@ -833,7 +837,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
|
|
|
833
837
|
datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
|
|
834
838
|
datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
|
|
835
839
|
datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
|
|
836
|
-
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=
|
|
840
|
+
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=HKZisxW2HNm6kwJUBaAVol7suJoxSVCjfvWy55HhalQ,11999
|
|
837
841
|
datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
|
|
838
842
|
datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
|
|
839
843
|
datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
|
|
@@ -942,7 +946,7 @@ datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
|
|
|
942
946
|
datahub/sdk/dataset.py,sha256=2-iD-HqjgFpCBmU3k8O5tkltmwFj4jaumADiX8sexJU,29465
|
|
943
947
|
datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
|
|
944
948
|
datahub/sdk/entity_client.py,sha256=xHLGLn6oJfxmHLXo5w4-isPFZGcf8yR5IwyC6pvl_I8,8903
|
|
945
|
-
datahub/sdk/lineage_client.py,sha256=
|
|
949
|
+
datahub/sdk/lineage_client.py,sha256=xHTX3B1KnVIh6vQLtJHGPMT6BXXru25UE1IK2T7RB28,33914
|
|
946
950
|
datahub/sdk/main_client.py,sha256=kJRmydyyaOjUdqOljl0w_Cx7BKkViZmnX-1vZBHY4nw,5266
|
|
947
951
|
datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
|
|
948
952
|
datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
|
|
@@ -976,10 +980,10 @@ datahub/sql_parsing/fingerprint_utils.py,sha256=3hGiexaQXnE7eZLxo-t7hlTyVQz7womb
|
|
|
976
980
|
datahub/sql_parsing/query_types.py,sha256=FKjDzszZzsrCfYfm7dgD6T_8865qxWl767fdGyHWBh4,2720
|
|
977
981
|
datahub/sql_parsing/schema_resolver.py,sha256=ISuingLcQnOJZkNXBkc73uPwYUbbOtERAjgGhJajDiQ,10782
|
|
978
982
|
datahub/sql_parsing/split_statements.py,sha256=OIQXA9e4k3G9Z1y7rbgdtZhMWt4FPnq41cE8Jkm9cBY,9542
|
|
979
|
-
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=
|
|
983
|
+
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=CmvBqCkIkeX606-uZv44N9meL8TnRydKpWFia-_CoEQ,71642
|
|
980
984
|
datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
|
|
981
985
|
datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
|
|
982
|
-
datahub/sql_parsing/sqlglot_lineage.py,sha256=
|
|
986
|
+
datahub/sql_parsing/sqlglot_lineage.py,sha256=IeWxkgOp7wP83Id_0NHy0Ra_L-CmjIs0Hm2ul_TWu7c,60003
|
|
983
987
|
datahub/sql_parsing/sqlglot_utils.py,sha256=TI11oBu1wrGeUuUGBg7hGTr6lTvztahdqiqXNJYRfbQ,14823
|
|
984
988
|
datahub/sql_parsing/tool_meta_extractor.py,sha256=EV_g7sOchTSUm2p6wluNJqND7-rDYokVTqqFCM7hQ6c,7599
|
|
985
989
|
datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -1034,7 +1038,7 @@ datahub/utilities/sentinels.py,sha256=p299J2YR62K2Mbuy-2lrGH4BnF9dkRqGo59Wub0hYV
|
|
|
1034
1038
|
datahub/utilities/serialized_lru_cache.py,sha256=R9enyN5BlatknX4KFq9euYc2yiE1rRMucPbAHg09Goc,3247
|
|
1035
1039
|
datahub/utilities/server_config_util.py,sha256=hUjPaA96eQrur6UqSAj7c5lPfLvyqcgKLFOrR7XG3EY,9618
|
|
1036
1040
|
datahub/utilities/sql_formatter.py,sha256=tYXIsKjKmpKh0JXGxeAPrHkUWYd1SwJNLjUZsfQP2h0,1016
|
|
1037
|
-
datahub/utilities/sqlalchemy_query_combiner.py,sha256=
|
|
1041
|
+
datahub/utilities/sqlalchemy_query_combiner.py,sha256=ZWGPG5mrKuumSSEQMrUSjUjSKTaDXw71KWI7Ytnv2xE,15027
|
|
1038
1042
|
datahub/utilities/sqlalchemy_type_converter.py,sha256=H4S4xnnyPozDBHFhBh4rjjoXa5novFzYIUBJy2KSrVc,9805
|
|
1039
1043
|
datahub/utilities/sqllineage_patch.py,sha256=0Buh50bmEqJFg1HFRCknCnePo1cecI4JmGxVhM_jh2g,1976
|
|
1040
1044
|
datahub/utilities/stats_collections.py,sha256=9QDEk40UxhmQwDS6I63Gp6fcIBqmXVinKl7x2xHCD34,1702
|
|
@@ -1087,8 +1091,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1087
1091
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1088
1092
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1089
1093
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1090
|
-
acryl_datahub-1.1.0.
|
|
1091
|
-
acryl_datahub-1.1.0.
|
|
1092
|
-
acryl_datahub-1.1.0.
|
|
1093
|
-
acryl_datahub-1.1.0.
|
|
1094
|
-
acryl_datahub-1.1.0.
|
|
1094
|
+
acryl_datahub-1.1.0.5rc7.dist-info/METADATA,sha256=OJxr8Zm0_d6AjnEsFM8eYv7CsCoyeKpj0ifhszN5uYc,181944
|
|
1095
|
+
acryl_datahub-1.1.0.5rc7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1096
|
+
acryl_datahub-1.1.0.5rc7.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
|
|
1097
|
+
acryl_datahub-1.1.0.5rc7.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1098
|
+
acryl_datahub-1.1.0.5rc7.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
datahub/ingestion/api/report.py
CHANGED
|
@@ -6,19 +6,25 @@ from collections import defaultdict
|
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
from datetime import datetime, timedelta
|
|
8
8
|
from enum import Enum
|
|
9
|
-
from typing import Any, Dict, Optional, Set, cast, runtime_checkable
|
|
9
|
+
from typing import Any, Dict, List, Optional, Set, Union, cast, runtime_checkable
|
|
10
10
|
|
|
11
11
|
import humanfriendly
|
|
12
12
|
import pydantic
|
|
13
13
|
from pydantic import BaseModel
|
|
14
14
|
from typing_extensions import Literal, Protocol
|
|
15
15
|
|
|
16
|
+
from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
|
16
17
|
from datahub.emitter.mcp_builder import mcps_from_mce
|
|
17
18
|
from datahub.ingestion.api.closeable import Closeable
|
|
18
19
|
from datahub.ingestion.api.report_helpers import format_datetime_relative
|
|
19
20
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
21
|
+
from datahub.ingestion.autogenerated.lineage_helper import is_lineage_aspect
|
|
20
22
|
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
|
|
21
|
-
from datahub.metadata.schema_classes import
|
|
23
|
+
from datahub.metadata.schema_classes import (
|
|
24
|
+
MetadataChangeProposalClass,
|
|
25
|
+
SubTypesClass,
|
|
26
|
+
UpstreamLineageClass,
|
|
27
|
+
)
|
|
22
28
|
from datahub.utilities.file_backed_collections import FileBackedDict
|
|
23
29
|
from datahub.utilities.lossy_collections import LossyList
|
|
24
30
|
|
|
@@ -125,8 +131,6 @@ class ReportAttribute(BaseModel):
|
|
|
125
131
|
|
|
126
132
|
@dataclass
|
|
127
133
|
class ExamplesReport(Report, Closeable):
|
|
128
|
-
_urns_seen: Set[str] = field(default_factory=set)
|
|
129
|
-
entities: Dict[str, list] = field(default_factory=lambda: defaultdict(LossyList))
|
|
130
134
|
aspects: Dict[str, Dict[str, int]] = field(
|
|
131
135
|
default_factory=lambda: defaultdict(lambda: defaultdict(int))
|
|
132
136
|
)
|
|
@@ -135,11 +139,16 @@ class ExamplesReport(Report, Closeable):
|
|
|
135
139
|
lambda: defaultdict(lambda: defaultdict(int))
|
|
136
140
|
)
|
|
137
141
|
)
|
|
138
|
-
|
|
139
|
-
default_factory=lambda: defaultdict(lambda: defaultdict(
|
|
142
|
+
samples: Dict[str, Dict[str, List[str]]] = field(
|
|
143
|
+
default_factory=lambda: defaultdict(lambda: defaultdict(list))
|
|
140
144
|
)
|
|
141
145
|
_file_based_dict: Optional[FileBackedDict[SourceReportSubtypes]] = None
|
|
142
146
|
|
|
147
|
+
# We are adding this to make querying easier for fine-grained lineage
|
|
148
|
+
_fine_grained_lineage_special_case_name = "fineGrainedLineages"
|
|
149
|
+
_samples_to_add: int = 20
|
|
150
|
+
_lineage_aspects_seen: Set[str] = field(default_factory=set)
|
|
151
|
+
|
|
143
152
|
def __post_init__(self) -> None:
|
|
144
153
|
self._file_based_dict = FileBackedDict(
|
|
145
154
|
tablename="urn_aspects",
|
|
@@ -157,6 +166,151 @@ class ExamplesReport(Report, Closeable):
|
|
|
157
166
|
self._file_based_dict.close()
|
|
158
167
|
self._file_based_dict = None
|
|
159
168
|
|
|
169
|
+
def _build_aspects_where_clause(self, aspects: List[str]) -> str:
|
|
170
|
+
"""Build WHERE clause for matching any of the given aspects."""
|
|
171
|
+
if not aspects:
|
|
172
|
+
return ""
|
|
173
|
+
|
|
174
|
+
conditions = []
|
|
175
|
+
for aspect in aspects:
|
|
176
|
+
conditions.append(f"aspects LIKE '%{aspect}%'")
|
|
177
|
+
|
|
178
|
+
return " OR ".join(conditions)
|
|
179
|
+
|
|
180
|
+
def _collect_samples_by_subtype(self, where_clause: str, sample_key: str) -> None:
|
|
181
|
+
"""Helper method to collect samples organized by subtype for a given where clause."""
|
|
182
|
+
|
|
183
|
+
subtype_query = f"""
|
|
184
|
+
SELECT DISTINCT subTypes
|
|
185
|
+
FROM urn_aspects
|
|
186
|
+
WHERE {where_clause}
|
|
187
|
+
"""
|
|
188
|
+
assert self._file_based_dict is not None
|
|
189
|
+
subtypes = set()
|
|
190
|
+
for row in self._file_based_dict.sql_query(subtype_query):
|
|
191
|
+
sub_type = row["subTypes"] or "unknown"
|
|
192
|
+
subtypes.add(sub_type)
|
|
193
|
+
|
|
194
|
+
for sub_type in subtypes:
|
|
195
|
+
query = f"""
|
|
196
|
+
SELECT urn
|
|
197
|
+
FROM urn_aspects
|
|
198
|
+
WHERE {where_clause} AND subTypes = ?
|
|
199
|
+
limit {self._samples_to_add}
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
for row in self._file_based_dict.sql_query(query, (sub_type,)):
|
|
203
|
+
self.samples[sample_key][sub_type].append(row["urn"])
|
|
204
|
+
|
|
205
|
+
def _collect_samples_by_aspects(self, aspects: List[str], sample_key: str) -> None:
|
|
206
|
+
"""Helper method to collect samples for entities that have any of the given aspects."""
|
|
207
|
+
if not aspects:
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
where_clause = self._build_aspects_where_clause(aspects)
|
|
211
|
+
self._collect_samples_by_subtype(where_clause, sample_key)
|
|
212
|
+
|
|
213
|
+
def _collect_samples_by_lineage_aspects(
|
|
214
|
+
self, aspects: List[str], sample_key: str
|
|
215
|
+
) -> None:
|
|
216
|
+
"""Helper method to collect samples for entities that have any of the given lineage aspects.
|
|
217
|
+
|
|
218
|
+
Lineage aspects are stored in JSON format and require quote escaping in LIKE clauses.
|
|
219
|
+
"""
|
|
220
|
+
if not aspects:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
lineage_conditions = []
|
|
224
|
+
for aspect in aspects:
|
|
225
|
+
lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
|
|
226
|
+
|
|
227
|
+
where_clause = " OR ".join(lineage_conditions)
|
|
228
|
+
self._collect_samples_by_subtype(where_clause, sample_key)
|
|
229
|
+
|
|
230
|
+
def _collect_samples_with_all_conditions(self, sample_key: str) -> None:
|
|
231
|
+
"""
|
|
232
|
+
Collect samples for entities that have lineage, profiling, and usage aspects.
|
|
233
|
+
These specific 3 cases are added here as these URNs will be shown in the UI. Subject to change in future.
|
|
234
|
+
"""
|
|
235
|
+
if not self._lineage_aspects_seen:
|
|
236
|
+
return
|
|
237
|
+
assert self._file_based_dict is not None
|
|
238
|
+
|
|
239
|
+
# Build lineage conditions using the same logic as _collect_samples_by_lineage_aspects
|
|
240
|
+
lineage_conditions = []
|
|
241
|
+
for aspect in self._lineage_aspects_seen:
|
|
242
|
+
lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
|
|
243
|
+
lineage_where_clause = " OR ".join(lineage_conditions)
|
|
244
|
+
|
|
245
|
+
# Build profiling conditions using the same logic as _collect_samples_by_aspects
|
|
246
|
+
profiling_where_clause = self._build_aspects_where_clause(["datasetProfile"])
|
|
247
|
+
|
|
248
|
+
# Build usage conditions using the same logic as _collect_samples_by_aspects
|
|
249
|
+
usage_where_clause = self._build_aspects_where_clause(
|
|
250
|
+
[
|
|
251
|
+
"datasetUsageStatistics",
|
|
252
|
+
"chartUsageStatistics",
|
|
253
|
+
"dashboardUsageStatistics",
|
|
254
|
+
]
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
query = f"""
|
|
258
|
+
SELECT urn, subTypes
|
|
259
|
+
FROM urn_aspects
|
|
260
|
+
WHERE ({lineage_where_clause})
|
|
261
|
+
AND ({profiling_where_clause})
|
|
262
|
+
AND ({usage_where_clause})
|
|
263
|
+
limit {self._samples_to_add}
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
for row in self._file_based_dict.sql_query(query):
|
|
267
|
+
sub_type = row["subTypes"] or "unknown"
|
|
268
|
+
self.samples[sample_key][sub_type].append(row["urn"])
|
|
269
|
+
|
|
270
|
+
def _has_fine_grained_lineage(
|
|
271
|
+
self, mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper]
|
|
272
|
+
) -> bool:
|
|
273
|
+
if isinstance(mcp.aspect, UpstreamLineageClass):
|
|
274
|
+
upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
|
|
275
|
+
if upstream_lineage.fineGrainedLineages:
|
|
276
|
+
return True
|
|
277
|
+
return False
|
|
278
|
+
|
|
279
|
+
def _update_file_based_dict(
|
|
280
|
+
self,
|
|
281
|
+
urn: str,
|
|
282
|
+
entityType: str,
|
|
283
|
+
aspectName: str,
|
|
284
|
+
mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper],
|
|
285
|
+
) -> None:
|
|
286
|
+
if is_lineage_aspect(entityType, aspectName):
|
|
287
|
+
self._lineage_aspects_seen.add(aspectName)
|
|
288
|
+
has_fine_grained_lineage = self._has_fine_grained_lineage(mcp)
|
|
289
|
+
|
|
290
|
+
sub_type = "unknown"
|
|
291
|
+
if isinstance(mcp.aspect, SubTypesClass):
|
|
292
|
+
sub_type = mcp.aspect.typeNames[0]
|
|
293
|
+
|
|
294
|
+
assert self._file_based_dict is not None
|
|
295
|
+
if urn in self._file_based_dict:
|
|
296
|
+
if sub_type != "unknown":
|
|
297
|
+
self._file_based_dict[urn].subType = sub_type
|
|
298
|
+
self._file_based_dict[urn].aspects.add(aspectName)
|
|
299
|
+
if has_fine_grained_lineage:
|
|
300
|
+
self._file_based_dict[urn].aspects.add(
|
|
301
|
+
self._fine_grained_lineage_special_case_name
|
|
302
|
+
)
|
|
303
|
+
self._file_based_dict.mark_dirty(urn)
|
|
304
|
+
else:
|
|
305
|
+
self._file_based_dict[urn] = SourceReportSubtypes(
|
|
306
|
+
urn=urn,
|
|
307
|
+
entity_type=entityType,
|
|
308
|
+
subType=sub_type,
|
|
309
|
+
aspects={aspectName}
|
|
310
|
+
if not has_fine_grained_lineage
|
|
311
|
+
else {aspectName, self._fine_grained_lineage_special_case_name},
|
|
312
|
+
)
|
|
313
|
+
|
|
160
314
|
def _store_workunit_data(self, wu: MetadataWorkUnit) -> None:
|
|
161
315
|
urn = wu.get_urn()
|
|
162
316
|
|
|
@@ -169,41 +323,15 @@ class ExamplesReport(Report, Closeable):
|
|
|
169
323
|
entityType = mcp.entityType
|
|
170
324
|
aspectName = mcp.aspectName
|
|
171
325
|
|
|
172
|
-
if urn not in self._urns_seen:
|
|
173
|
-
self._urns_seen.add(urn)
|
|
174
|
-
self.entities[entityType].append(urn)
|
|
175
|
-
|
|
176
326
|
if aspectName is None:
|
|
177
327
|
continue
|
|
178
|
-
|
|
179
|
-
self.
|
|
180
|
-
sub_type = "unknown"
|
|
181
|
-
if isinstance(mcp.aspect, UpstreamLineageClass):
|
|
182
|
-
upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
|
|
183
|
-
if upstream_lineage.fineGrainedLineages:
|
|
184
|
-
self.aspect_urn_samples[entityType]["fineGrainedLineages"].append(
|
|
185
|
-
urn
|
|
186
|
-
)
|
|
187
|
-
self.aspects[entityType]["fineGrainedLineages"] += 1
|
|
188
|
-
elif isinstance(mcp.aspect, SubTypesClass):
|
|
189
|
-
sub_type = mcp.aspect.typeNames[0]
|
|
190
|
-
assert self._file_based_dict is not None
|
|
191
|
-
if urn in self._file_based_dict:
|
|
192
|
-
if sub_type != "unknown":
|
|
193
|
-
self._file_based_dict[urn].subType = sub_type
|
|
194
|
-
self._file_based_dict[urn].aspects.add(aspectName)
|
|
195
|
-
self._file_based_dict.mark_dirty(urn)
|
|
196
|
-
else:
|
|
197
|
-
self._file_based_dict[urn] = SourceReportSubtypes(
|
|
198
|
-
urn=urn,
|
|
199
|
-
entity_type=entityType,
|
|
200
|
-
subType=sub_type,
|
|
201
|
-
aspects={aspectName},
|
|
202
|
-
)
|
|
328
|
+
|
|
329
|
+
self._update_file_based_dict(urn, entityType, aspectName, mcp)
|
|
203
330
|
|
|
204
331
|
def compute_stats(self) -> None:
|
|
205
332
|
if self._file_based_dict is None:
|
|
206
333
|
return
|
|
334
|
+
|
|
207
335
|
query = """
|
|
208
336
|
SELECT entityType, subTypes, aspects, count(*) as count
|
|
209
337
|
FROM urn_aspects
|
|
@@ -223,11 +351,31 @@ class ExamplesReport(Report, Closeable):
|
|
|
223
351
|
for aspect in aspects:
|
|
224
352
|
entity_subtype_aspect_counts[entity_type][sub_type][aspect] += count
|
|
225
353
|
|
|
354
|
+
self.aspects.clear()
|
|
226
355
|
self.aspects_by_subtypes.clear()
|
|
356
|
+
_aspects_seen: Set[str] = set()
|
|
227
357
|
for entity_type, subtype_counts in entity_subtype_aspect_counts.items():
|
|
228
358
|
for sub_type, aspect_counts in subtype_counts.items():
|
|
359
|
+
for aspect, count in aspect_counts.items():
|
|
360
|
+
self.aspects[entity_type][aspect] += count
|
|
361
|
+
_aspects_seen.add(aspect)
|
|
229
362
|
self.aspects_by_subtypes[entity_type][sub_type] = dict(aspect_counts)
|
|
230
363
|
|
|
364
|
+
self.samples.clear()
|
|
365
|
+
self._collect_samples_by_aspects(["datasetProfile"], "profiling")
|
|
366
|
+
self._collect_samples_by_aspects(
|
|
367
|
+
[
|
|
368
|
+
"datasetUsageStatistics",
|
|
369
|
+
"chartUsageStatistics",
|
|
370
|
+
"dashboardUsageStatistics",
|
|
371
|
+
],
|
|
372
|
+
"usage",
|
|
373
|
+
)
|
|
374
|
+
self._collect_samples_by_lineage_aspects(
|
|
375
|
+
list(self._lineage_aspects_seen), "lineage"
|
|
376
|
+
)
|
|
377
|
+
self._collect_samples_with_all_conditions("all_3")
|
|
378
|
+
|
|
231
379
|
|
|
232
380
|
class EntityFilterReport(ReportAttribute):
|
|
233
381
|
type: str
|