acryl-datahub 1.1.0.5rc6__py3-none-any.whl → 1.1.0.5rc7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/METADATA +2547 -2547
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/RECORD +38 -35
- datahub/_version.py +1 -1
- datahub/ingestion/api/report.py +183 -35
- datahub/ingestion/autogenerated/capability_summary.json +3366 -0
- datahub/ingestion/autogenerated/lineage.json +401 -0
- datahub/ingestion/autogenerated/lineage_helper.py +30 -128
- datahub/ingestion/run/pipeline.py +4 -1
- datahub/ingestion/source/bigquery_v2/bigquery.py +23 -22
- datahub/ingestion/source/cassandra/cassandra_profiling.py +6 -5
- datahub/ingestion/source/common/subtypes.py +1 -1
- datahub/ingestion/source/data_lake_common/object_store.py +40 -0
- datahub/ingestion/source/dremio/dremio_source.py +6 -3
- datahub/ingestion/source/gcs/gcs_source.py +4 -1
- datahub/ingestion/source/ge_data_profiler.py +28 -20
- datahub/ingestion/source/kafka_connect/source_connectors.py +59 -4
- datahub/ingestion/source/mock_data/datahub_mock_data.py +45 -0
- datahub/ingestion/source/redshift/usage.py +4 -3
- datahub/ingestion/source/s3/source.py +19 -3
- datahub/ingestion/source/snowflake/snowflake_queries.py +47 -3
- datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
- datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
- datahub/ingestion/source/unity/proxy.py +4 -3
- datahub/ingestion/source/unity/source.py +10 -8
- datahub/integrations/assertion/snowflake/compiler.py +4 -3
- datahub/metadata/_internal_schema_classes.py +85 -4
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +2 -0
- datahub/metadata/schema.avsc +54 -1
- datahub/metadata/schemas/CorpUserSettings.avsc +17 -1
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +37 -0
- datahub/sdk/lineage_client.py +2 -0
- datahub/sql_parsing/sql_parsing_aggregator.py +3 -3
- datahub/sql_parsing/sqlglot_lineage.py +2 -0
- datahub/utilities/sqlalchemy_query_combiner.py +5 -2
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.1.0.
|
|
1
|
+
acryl_datahub-1.1.0.5rc7.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=sUspZGha0FX624vViKEbY5GTEpcVIfY_B0Gq4roHDZ0,323
|
|
5
5
|
datahub/entrypoints.py,sha256=H-YFTvxTJOgpWsFBVlxyb1opjkq-hjTzNmjy5Fq3RHg,8992
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -148,7 +148,7 @@ datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPs
|
|
|
148
148
|
datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
|
|
149
149
|
datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
|
|
150
150
|
datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
|
|
151
|
-
datahub/ingestion/api/report.py,sha256=
|
|
151
|
+
datahub/ingestion/api/report.py,sha256=SRZ1DNyHUV08uxA3qGzWOX-lVfSSwh4ln6NOSwMYhIA,14807
|
|
152
152
|
datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
|
|
153
153
|
datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
|
|
154
154
|
datahub/ingestion/api/source.py,sha256=WcpmuWRua87_UbaSY8GtkLkC0Gql0pM_i5FJN5ALh88,19669
|
|
@@ -159,7 +159,9 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
|
159
159
|
datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
|
|
160
160
|
datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=fMjPnyWEofIZV52E2AFYU3IgBJwyZvbygXxCJyEtcWI,4442
|
|
161
161
|
datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
datahub/ingestion/autogenerated/
|
|
162
|
+
datahub/ingestion/autogenerated/capability_summary.json,sha256=CiZ0OZlU40jaItCrOYwdUfnb2JVZSb-KYBr2-MiMfpc,104572
|
|
163
|
+
datahub/ingestion/autogenerated/lineage.json,sha256=wjaXxrzpo3M9ppc80TKIgCTuCg0jpb-D9MhBsHXaMOY,10004
|
|
164
|
+
datahub/ingestion/autogenerated/lineage_helper.py,sha256=MJrHGuDvM_9hrFfRt0dkzE58j1xIsvgW6UIQBEKqdYE,2701
|
|
163
165
|
datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
164
166
|
datahub/ingestion/extractor/extractor_registry.py,sha256=f7CLfW3pr29QZkXSHbp7HjUrsdw7ejQJmot-tiSPcqc,342
|
|
165
167
|
datahub/ingestion/extractor/json_ref_patch.py,sha256=4g3ZWHn7rwS74jUvSXJiGpi-UKHhiSYKKgBeU4E5ukE,1448
|
|
@@ -191,7 +193,7 @@ datahub/ingestion/reporting/file_reporter.py,sha256=tiWukmMxHrTQI3rOAumsq6lRlw8T
|
|
|
191
193
|
datahub/ingestion/reporting/reporting_provider_registry.py,sha256=jTYSh3T4sensjnHQfPLiIcbA2dG8w0px9ghChAJjGdU,310
|
|
192
194
|
datahub/ingestion/run/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
193
195
|
datahub/ingestion/run/connection.py,sha256=mngNzr5aRLUDa5Izqxa0xkdDEqEqcDuacWSKIlkdvPc,1483
|
|
194
|
-
datahub/ingestion/run/pipeline.py,sha256=
|
|
196
|
+
datahub/ingestion/run/pipeline.py,sha256=FGRvCo9aRIMtDa4a3JHy1u2xi70p9TQuiiC17yXZVSc,30287
|
|
195
197
|
datahub/ingestion/run/pipeline_config.py,sha256=joG1j9OlwJhb8zqv4TY6_FSzOaKOx6xsBu255A5lP8g,4101
|
|
196
198
|
datahub/ingestion/run/sink_callback.py,sha256=xZAzaKkR0dcQP838pMJWsA52oaQXV5BiqXkpxEpJ_9U,2856
|
|
197
199
|
datahub/ingestion/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -209,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
|
|
|
209
211
|
datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
|
|
210
212
|
datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
|
|
211
213
|
datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
|
|
212
|
-
datahub/ingestion/source/ge_data_profiler.py,sha256=
|
|
214
|
+
datahub/ingestion/source/ge_data_profiler.py,sha256=TnAupfORYFzcgP7akQEqeuvy1qO0sxMetb0B75f95jM,67980
|
|
213
215
|
datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
|
|
214
216
|
datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
|
|
215
217
|
datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
|
|
@@ -254,7 +256,7 @@ datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0
|
|
|
254
256
|
datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
|
|
255
257
|
datahub/ingestion/source/azure/azure_common.py,sha256=Zl0pPuE6L3QcM5B1P0LsPthZmD0h7fUUS0kg2okl6IY,4053
|
|
256
258
|
datahub/ingestion/source/bigquery_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
257
|
-
datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=
|
|
259
|
+
datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=QNG_3IU6xFyGz7Xq2dHyPS0C-LCuJyMAFpqPrMLRBsE,15100
|
|
258
260
|
datahub/ingestion/source/bigquery_v2/bigquery_audit.py,sha256=kEwWhq3ch6WT4q4hcX8-fvQh28KgrNfspFwIytO3vQA,25103
|
|
259
261
|
datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py,sha256=LuGJ6LgPViLIfDQfylxlQ3CA7fZYM5MDt8M-7sfzm84,5096
|
|
260
262
|
datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=bG1soRawCLhJw_15L2fJmFfj1kntTthV6ng4LZOnwko,21916
|
|
@@ -277,17 +279,17 @@ datahub/ingestion/source/cassandra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
|
|
|
277
279
|
datahub/ingestion/source/cassandra/cassandra.py,sha256=k2EKpeUUZ0O_RwQ6NXM15vnTsY3OgST4e5LG8Eg6Coo,14534
|
|
278
280
|
datahub/ingestion/source/cassandra/cassandra_api.py,sha256=b7MApc3_tEfHoj-6ub6snkcv_DweL1wi_TGJjAA1-yU,13516
|
|
279
281
|
datahub/ingestion/source/cassandra/cassandra_config.py,sha256=Ga9915cDZukR5-u2tMNx5Jkf8eza2oAE5YS_sQIVEVQ,4222
|
|
280
|
-
datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=
|
|
282
|
+
datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=FdgPK_8s8otTOJDqNM4rpF6Mn4lFWbnjTaKEChzn2iE,11011
|
|
281
283
|
datahub/ingestion/source/cassandra/cassandra_utils.py,sha256=j-LidYkaCTmGnpUVNLsax_c3z32PsQbsbHeYojygd1s,5105
|
|
282
284
|
datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
283
285
|
datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
|
|
284
286
|
datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
|
|
285
287
|
datahub/ingestion/source/common/gcp_credentials_config.py,sha256=_NapGkAqZMbXNClLlmOfANS7U6rChhdthRX9s9iUv9k,2411
|
|
286
|
-
datahub/ingestion/source/common/subtypes.py,sha256
|
|
288
|
+
datahub/ingestion/source/common/subtypes.py,sha256=-z-QseXQC-Pt8pbbw4CsMRsqIcNzZ-mz-9iQSXyU65s,4493
|
|
287
289
|
datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
288
290
|
datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
|
|
289
291
|
datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGDTUOol9Oi3jZs8Pb04PIsHdC2WPP0dkg,6576
|
|
290
|
-
datahub/ingestion/source/data_lake_common/object_store.py,sha256=
|
|
292
|
+
datahub/ingestion/source/data_lake_common/object_store.py,sha256=NkbCXq56sdGkyBAxcrpSVE6GySLrh9E2jYZU6lYPupM,21644
|
|
291
293
|
datahub/ingestion/source/data_lake_common/path_spec.py,sha256=ekJAr4-PE2RhzQnmKb2xcSs_YncC1Dz95-UrXI67Vos,23584
|
|
292
294
|
datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
293
295
|
datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
|
|
@@ -317,7 +319,7 @@ datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py,sha256=MQk8BAHL
|
|
|
317
319
|
datahub/ingestion/source/dremio/dremio_entities.py,sha256=1gZrNqTp3Pm6vqGDQaWt3HkxEuHKxpGYQ4geVoFvxWI,15147
|
|
318
320
|
datahub/ingestion/source/dremio/dremio_profiling.py,sha256=TAcnpo8ZRKhLDHnQSJzJg3YdwTSyEa73LUAzENs7wG4,12287
|
|
319
321
|
datahub/ingestion/source/dremio/dremio_reporting.py,sha256=YRKM6PvoJYHLBXmOGwkgou_8x8_oA2xaqTWWoVuwFMY,2247
|
|
320
|
-
datahub/ingestion/source/dremio/dremio_source.py,sha256=
|
|
322
|
+
datahub/ingestion/source/dremio/dremio_source.py,sha256=ODIUhDr6Nx5ziV7Buo-ayT40WxkSKNhFhOlM6fmw-xM,25236
|
|
321
323
|
datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=wA1hqKk9cKMJDyEdZRQcDDLZPGYwuNqrvleUHTkWgrQ,10508
|
|
322
324
|
datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
323
325
|
datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
|
|
@@ -334,7 +336,7 @@ datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=mUWcMt-_FL1SYGIgI4lGZD
|
|
|
334
336
|
datahub/ingestion/source/gc/execution_request_cleanup.py,sha256=y-9ZIs_DZPUzYH1CI6HmaAZg3olNNA7MjT8HrCqAI0k,11159
|
|
335
337
|
datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py,sha256=4-qQR_2HGIYU8kC2hRIsJyKKMb9lKq4B6paJm_abUk4,12628
|
|
336
338
|
datahub/ingestion/source/gcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
337
|
-
datahub/ingestion/source/gcs/gcs_source.py,sha256=
|
|
339
|
+
datahub/ingestion/source/gcs/gcs_source.py,sha256=mrecg2ewT2ATrUuEYg3cMLPXXAWwc2WJzJFmAHNWuTc,7063
|
|
338
340
|
datahub/ingestion/source/gcs/gcs_utils.py,sha256=Kd2usZYIMFeSuE6_tJ4OoHGOdvG8mWaScFuAcIkC6P0,1789
|
|
339
341
|
datahub/ingestion/source/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
340
342
|
datahub/ingestion/source/git/git_import.py,sha256=5CT6vMDb0MDctCtShnxb3JVihULtvkYGr9judHJFsOk,4143
|
|
@@ -362,7 +364,7 @@ datahub/ingestion/source/kafka_connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
|
|
|
362
364
|
datahub/ingestion/source/kafka_connect/common.py,sha256=lH64n1v_rJamWGfidBeuQJj8W1_IvOBpXQLR2YZaEvQ,7057
|
|
363
365
|
datahub/ingestion/source/kafka_connect/kafka_connect.py,sha256=AVAgBvgH7kM9I2ke3mwr8CfIL1J2SdVHH_86rnCFwrM,17727
|
|
364
366
|
datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfICTXIsv3ZfzlrMA4VlZXKQTcFCyIAU,17220
|
|
365
|
-
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=
|
|
367
|
+
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
|
|
366
368
|
datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
367
369
|
datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
|
|
368
370
|
datahub/ingestion/source/looker/looker_common.py,sha256=wwCRHyd0F799CEBboryQ4TZV057krnY2nMJOkUWc268,64765
|
|
@@ -389,7 +391,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
|
|
|
389
391
|
datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
|
|
390
392
|
datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
|
|
391
393
|
datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
392
|
-
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=
|
|
394
|
+
datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=PLQU1WnxDljZJGifoh6CqIQdBZrQAeBxtV-9NLdxM8o,15847
|
|
393
395
|
datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
|
|
394
396
|
datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
|
|
395
397
|
datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -438,13 +440,13 @@ datahub/ingestion/source/redshift/redshift.py,sha256=p6rOOCjxNnPpTn-vFjgISMMjtUT
|
|
|
438
440
|
datahub/ingestion/source/redshift/redshift_data_reader.py,sha256=zc69jwXHdF-w8J4Hq-ZQ6BjHQ75Ij2iNDMpoRJlcmlU,1724
|
|
439
441
|
datahub/ingestion/source/redshift/redshift_schema.py,sha256=7F-l_omOuKMuGE_rBWXVPG_GWXFKnCMzC4frNxZB9cs,24800
|
|
440
442
|
datahub/ingestion/source/redshift/report.py,sha256=O3QFozHlmMbH9b7KxbqhgTgr_0tCryj6FIzMiN6kRxw,3044
|
|
441
|
-
datahub/ingestion/source/redshift/usage.py,sha256=
|
|
443
|
+
datahub/ingestion/source/redshift/usage.py,sha256=Q7R-caJovLXv33uZepMGX5Cvm4DqQSLZdiL_s-p06wU,17473
|
|
442
444
|
datahub/ingestion/source/s3/__init__.py,sha256=HjqFPj11WtNFZM3kcVshlDb7kOsc19-l_3LM8PBjlJM,56
|
|
443
445
|
datahub/ingestion/source/s3/config.py,sha256=lElFXgEpKDT9SVoiXvtx98wV6Gp880qP4pLQaOGJGOo,7828
|
|
444
446
|
datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
|
|
445
447
|
datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
|
|
446
448
|
datahub/ingestion/source/s3/report.py,sha256=9Ej1UCChw963UpGw1-7asi5vFrOM232gfgG8bRdKPp0,667
|
|
447
|
-
datahub/ingestion/source/s3/source.py,sha256=
|
|
449
|
+
datahub/ingestion/source/s3/source.py,sha256=DuJ5G0S2ukZFRL-aO8b4ihzhTlxvbzbj2_GyS8zLb7s,60968
|
|
448
450
|
datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
449
451
|
datahub/ingestion/source/sac/sac.py,sha256=0s_JxHGOhit3Wvgbg7qQi-Z9j9_TgBX_I1yOR3L6-rA,30243
|
|
450
452
|
datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
|
|
@@ -475,7 +477,7 @@ datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUH
|
|
|
475
477
|
datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
|
|
476
478
|
datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
|
|
477
479
|
datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
|
|
478
|
-
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=
|
|
480
|
+
datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=PHDFks4rhSfqsPLcamMZJDDJic8988P3WNWglXIUN6M,33017
|
|
479
481
|
datahub/ingestion/source/snowflake/snowflake_query.py,sha256=kqOxibplHyDhdioue8MeXBzeDS9d274-dspb-wyWMMI,38165
|
|
480
482
|
datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
|
|
481
483
|
datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
|
|
@@ -483,9 +485,10 @@ datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=_37-AQyI4uGt4f
|
|
|
483
485
|
datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
|
|
484
486
|
datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GKONtWCTtYvTdR209cRW9g66xyYE,5671
|
|
485
487
|
datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
|
|
486
|
-
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=
|
|
488
|
+
datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
|
|
487
489
|
datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=w-SftpjHSOf_6S2T2srHJyJta3MHS0usmn4Z9jgx4QE,13858
|
|
488
490
|
datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=eqOVRYmFiWQPAIDWUcOo5QZkG05xanlEbxz4aDTvSNM,34706
|
|
491
|
+
datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=XVEP8rGGIqds8f7RRoUHGajsT0myBpYS_u5aVdwggBU,5308
|
|
489
492
|
datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
490
493
|
datahub/ingestion/source/sql/athena.py,sha256=lQ_zniHtmtSFCnSmzM2dG7ND6zOZmhu6t-I4ZOeQKho,27652
|
|
491
494
|
datahub/ingestion/source/sql/athena_properties_extractor.py,sha256=n2SvqeUbNWxiWWdkDs8VYlUPlLwfZzZy9AIa-V4D7AY,28531
|
|
@@ -547,11 +550,11 @@ datahub/ingestion/source/unity/config.py,sha256=7QosoBthg9kirHfXev_vhefkobUxYnp1
|
|
|
547
550
|
datahub/ingestion/source/unity/connection_test.py,sha256=B143Wb28fS0V4GhygU9hzKqiArWBjsQO54IUCPf23dc,2586
|
|
548
551
|
datahub/ingestion/source/unity/ge_profiler.py,sha256=NBRHZceq-f95iUn7u0h7cgcd9nAc48Aa-lmp_BqE0As,8409
|
|
549
552
|
datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
|
|
550
|
-
datahub/ingestion/source/unity/proxy.py,sha256=
|
|
553
|
+
datahub/ingestion/source/unity/proxy.py,sha256=i15xEfjt7sbajbDIA9cbrxKWLByoL2drI8lnvWPceqc,22787
|
|
551
554
|
datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
|
|
552
555
|
datahub/ingestion/source/unity/proxy_types.py,sha256=qrvHiwPzl5cPX-KRvcIGGeJVdr0I8XUQmoAI6ErZ-v8,9371
|
|
553
556
|
datahub/ingestion/source/unity/report.py,sha256=XFT9oQfvEB4RkTvWGgFOoQuLPUN_AIoPXZ79xeDhGHQ,2831
|
|
554
|
-
datahub/ingestion/source/unity/source.py,sha256=
|
|
557
|
+
datahub/ingestion/source/unity/source.py,sha256=_cqxiQYZWyNXHpnpZ6EGO9WRfL6a_7fwLwv5cVZdto4,49826
|
|
555
558
|
datahub/ingestion/source/unity/tag_entities.py,sha256=iWl6nRAWSye1hoFDx_Xh4aT53PN0sGzlX7n1-oTVUv8,11568
|
|
556
559
|
datahub/ingestion/source/unity/usage.py,sha256=0wETBAaZvHI_EGgBlxX3bKsVHEAdnUV8_bKI_lbyWjY,11500
|
|
557
560
|
datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -599,7 +602,7 @@ datahub/integrations/assertion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
|
|
|
599
602
|
datahub/integrations/assertion/common.py,sha256=rUkcyJ87vedWqJPVGco0e6FPF07zj8DNOPW1knIvDg8,2297
|
|
600
603
|
datahub/integrations/assertion/registry.py,sha256=mmeYpQREFVrLWpLcA0qYZtbrMX4vAGcDoQ59wtMgEcg,307
|
|
601
604
|
datahub/integrations/assertion/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
602
|
-
datahub/integrations/assertion/snowflake/compiler.py,sha256=
|
|
605
|
+
datahub/integrations/assertion/snowflake/compiler.py,sha256=qL757BVcpR9MoUOHPKBmDoid_5OSNbCUbbt3BivEds8,9970
|
|
603
606
|
datahub/integrations/assertion/snowflake/dmf_generator.py,sha256=ti4EnSlTGdAWmBcxgUqqilCGhLVGutpfSatbENP94O8,746
|
|
604
607
|
datahub/integrations/assertion/snowflake/field_metric_sql_generator.py,sha256=V2kwDTgg3ChnxW2h5pITpUwtCiwL9Lmhg4p6T5iS9IE,6513
|
|
605
608
|
datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py,sha256=aVVOhYPZKVITVNfGEQO54_XjjRuMplsu-yeFLE7MhxA,8950
|
|
@@ -615,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
|
|
|
615
618
|
datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
|
|
616
619
|
datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
|
|
617
620
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
618
|
-
datahub/metadata/_internal_schema_classes.py,sha256=
|
|
619
|
-
datahub/metadata/schema.avsc,sha256=
|
|
621
|
+
datahub/metadata/_internal_schema_classes.py,sha256=h_FmxHQdH1tChgFYOGgULr6RMgiXN9JfFOoMC3TSoEg,1043586
|
|
622
|
+
datahub/metadata/schema.avsc,sha256=zrngtDblYg2sLuu-5t5IJxRq52TexvIIElfoU99wuZw,720852
|
|
620
623
|
datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
|
|
621
624
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
622
625
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -687,7 +690,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schema/__init__.py,sha256=7JpzLs6S_Ey
|
|
|
687
690
|
datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWeznycKnHBfPEGcCHXPEz83Iq9ypjNaoSfeQeDU9g,397
|
|
688
691
|
datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
|
|
689
692
|
datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
|
|
690
|
-
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=
|
|
693
|
+
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=hKpTDXj2YEBja4hk2fZx3G9yzJ7eCFxcGCbbN0NydTk,933
|
|
691
694
|
datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
|
|
692
695
|
datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
|
|
693
696
|
datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
|
|
@@ -728,7 +731,7 @@ datahub/metadata/schemas/CorpUserCredentials.avsc,sha256=S7FkV9K_DGxhb4GFYbM5_lP
|
|
|
728
731
|
datahub/metadata/schemas/CorpUserEditableInfo.avsc,sha256=6IrqWidbHP7mRryfVlWAQU0JS34THHTM8_aIKWqClUE,3843
|
|
729
732
|
datahub/metadata/schemas/CorpUserInfo.avsc,sha256=oObOza-5FLjZyCjj0FN4MNV1DodgTwJSV4APduAggjk,3955
|
|
730
733
|
datahub/metadata/schemas/CorpUserKey.avsc,sha256=01sbbdr8G-ZP1yc2UfY3gR-YN6b7AvDbCbNpZJ-02J4,1025
|
|
731
|
-
datahub/metadata/schemas/CorpUserSettings.avsc,sha256=
|
|
734
|
+
datahub/metadata/schemas/CorpUserSettings.avsc,sha256=ff-rvW3ihwo6s9usvStx0aXTmCNmcR1TjggDxhbCacc,6529
|
|
732
735
|
datahub/metadata/schemas/CorpUserStatus.avsc,sha256=yqojAXEQ9CjRhY58RPyTUxzmFbHSANGGaMMbqiYZZIE,2538
|
|
733
736
|
datahub/metadata/schemas/Cost.avsc,sha256=o4kYZSss2uEwJ6gCA9fhBUoyD5xUqcSxz78vkIXXzGQ,1494
|
|
734
737
|
datahub/metadata/schemas/DashboardInfo.avsc,sha256=li2lSV6R4V-nz6foOi-NYxt_8ShHWfoKRw6M2BG5530,12907
|
|
@@ -834,7 +837,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
|
|
|
834
837
|
datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
|
|
835
838
|
datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
|
|
836
839
|
datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
|
|
837
|
-
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=
|
|
840
|
+
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=HKZisxW2HNm6kwJUBaAVol7suJoxSVCjfvWy55HhalQ,11999
|
|
838
841
|
datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
|
|
839
842
|
datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
|
|
840
843
|
datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
|
|
@@ -943,7 +946,7 @@ datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
|
|
|
943
946
|
datahub/sdk/dataset.py,sha256=2-iD-HqjgFpCBmU3k8O5tkltmwFj4jaumADiX8sexJU,29465
|
|
944
947
|
datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
|
|
945
948
|
datahub/sdk/entity_client.py,sha256=xHLGLn6oJfxmHLXo5w4-isPFZGcf8yR5IwyC6pvl_I8,8903
|
|
946
|
-
datahub/sdk/lineage_client.py,sha256=
|
|
949
|
+
datahub/sdk/lineage_client.py,sha256=xHTX3B1KnVIh6vQLtJHGPMT6BXXru25UE1IK2T7RB28,33914
|
|
947
950
|
datahub/sdk/main_client.py,sha256=kJRmydyyaOjUdqOljl0w_Cx7BKkViZmnX-1vZBHY4nw,5266
|
|
948
951
|
datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
|
|
949
952
|
datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
|
|
@@ -977,10 +980,10 @@ datahub/sql_parsing/fingerprint_utils.py,sha256=3hGiexaQXnE7eZLxo-t7hlTyVQz7womb
|
|
|
977
980
|
datahub/sql_parsing/query_types.py,sha256=FKjDzszZzsrCfYfm7dgD6T_8865qxWl767fdGyHWBh4,2720
|
|
978
981
|
datahub/sql_parsing/schema_resolver.py,sha256=ISuingLcQnOJZkNXBkc73uPwYUbbOtERAjgGhJajDiQ,10782
|
|
979
982
|
datahub/sql_parsing/split_statements.py,sha256=OIQXA9e4k3G9Z1y7rbgdtZhMWt4FPnq41cE8Jkm9cBY,9542
|
|
980
|
-
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=
|
|
983
|
+
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=CmvBqCkIkeX606-uZv44N9meL8TnRydKpWFia-_CoEQ,71642
|
|
981
984
|
datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
|
|
982
985
|
datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
|
|
983
|
-
datahub/sql_parsing/sqlglot_lineage.py,sha256=
|
|
986
|
+
datahub/sql_parsing/sqlglot_lineage.py,sha256=IeWxkgOp7wP83Id_0NHy0Ra_L-CmjIs0Hm2ul_TWu7c,60003
|
|
984
987
|
datahub/sql_parsing/sqlglot_utils.py,sha256=TI11oBu1wrGeUuUGBg7hGTr6lTvztahdqiqXNJYRfbQ,14823
|
|
985
988
|
datahub/sql_parsing/tool_meta_extractor.py,sha256=EV_g7sOchTSUm2p6wluNJqND7-rDYokVTqqFCM7hQ6c,7599
|
|
986
989
|
datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -1035,7 +1038,7 @@ datahub/utilities/sentinels.py,sha256=p299J2YR62K2Mbuy-2lrGH4BnF9dkRqGo59Wub0hYV
|
|
|
1035
1038
|
datahub/utilities/serialized_lru_cache.py,sha256=R9enyN5BlatknX4KFq9euYc2yiE1rRMucPbAHg09Goc,3247
|
|
1036
1039
|
datahub/utilities/server_config_util.py,sha256=hUjPaA96eQrur6UqSAj7c5lPfLvyqcgKLFOrR7XG3EY,9618
|
|
1037
1040
|
datahub/utilities/sql_formatter.py,sha256=tYXIsKjKmpKh0JXGxeAPrHkUWYd1SwJNLjUZsfQP2h0,1016
|
|
1038
|
-
datahub/utilities/sqlalchemy_query_combiner.py,sha256=
|
|
1041
|
+
datahub/utilities/sqlalchemy_query_combiner.py,sha256=ZWGPG5mrKuumSSEQMrUSjUjSKTaDXw71KWI7Ytnv2xE,15027
|
|
1039
1042
|
datahub/utilities/sqlalchemy_type_converter.py,sha256=H4S4xnnyPozDBHFhBh4rjjoXa5novFzYIUBJy2KSrVc,9805
|
|
1040
1043
|
datahub/utilities/sqllineage_patch.py,sha256=0Buh50bmEqJFg1HFRCknCnePo1cecI4JmGxVhM_jh2g,1976
|
|
1041
1044
|
datahub/utilities/stats_collections.py,sha256=9QDEk40UxhmQwDS6I63Gp6fcIBqmXVinKl7x2xHCD34,1702
|
|
@@ -1088,8 +1091,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1088
1091
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1089
1092
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1090
1093
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1091
|
-
acryl_datahub-1.1.0.
|
|
1092
|
-
acryl_datahub-1.1.0.
|
|
1093
|
-
acryl_datahub-1.1.0.
|
|
1094
|
-
acryl_datahub-1.1.0.
|
|
1095
|
-
acryl_datahub-1.1.0.
|
|
1094
|
+
acryl_datahub-1.1.0.5rc7.dist-info/METADATA,sha256=OJxr8Zm0_d6AjnEsFM8eYv7CsCoyeKpj0ifhszN5uYc,181944
|
|
1095
|
+
acryl_datahub-1.1.0.5rc7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1096
|
+
acryl_datahub-1.1.0.5rc7.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
|
|
1097
|
+
acryl_datahub-1.1.0.5rc7.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1098
|
+
acryl_datahub-1.1.0.5rc7.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
datahub/ingestion/api/report.py
CHANGED
|
@@ -6,19 +6,25 @@ from collections import defaultdict
|
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
from datetime import datetime, timedelta
|
|
8
8
|
from enum import Enum
|
|
9
|
-
from typing import Any, Dict, Optional, Set, cast, runtime_checkable
|
|
9
|
+
from typing import Any, Dict, List, Optional, Set, Union, cast, runtime_checkable
|
|
10
10
|
|
|
11
11
|
import humanfriendly
|
|
12
12
|
import pydantic
|
|
13
13
|
from pydantic import BaseModel
|
|
14
14
|
from typing_extensions import Literal, Protocol
|
|
15
15
|
|
|
16
|
+
from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
|
16
17
|
from datahub.emitter.mcp_builder import mcps_from_mce
|
|
17
18
|
from datahub.ingestion.api.closeable import Closeable
|
|
18
19
|
from datahub.ingestion.api.report_helpers import format_datetime_relative
|
|
19
20
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
21
|
+
from datahub.ingestion.autogenerated.lineage_helper import is_lineage_aspect
|
|
20
22
|
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
|
|
21
|
-
from datahub.metadata.schema_classes import
|
|
23
|
+
from datahub.metadata.schema_classes import (
|
|
24
|
+
MetadataChangeProposalClass,
|
|
25
|
+
SubTypesClass,
|
|
26
|
+
UpstreamLineageClass,
|
|
27
|
+
)
|
|
22
28
|
from datahub.utilities.file_backed_collections import FileBackedDict
|
|
23
29
|
from datahub.utilities.lossy_collections import LossyList
|
|
24
30
|
|
|
@@ -125,8 +131,6 @@ class ReportAttribute(BaseModel):
|
|
|
125
131
|
|
|
126
132
|
@dataclass
|
|
127
133
|
class ExamplesReport(Report, Closeable):
|
|
128
|
-
_urns_seen: Set[str] = field(default_factory=set)
|
|
129
|
-
entities: Dict[str, list] = field(default_factory=lambda: defaultdict(LossyList))
|
|
130
134
|
aspects: Dict[str, Dict[str, int]] = field(
|
|
131
135
|
default_factory=lambda: defaultdict(lambda: defaultdict(int))
|
|
132
136
|
)
|
|
@@ -135,11 +139,16 @@ class ExamplesReport(Report, Closeable):
|
|
|
135
139
|
lambda: defaultdict(lambda: defaultdict(int))
|
|
136
140
|
)
|
|
137
141
|
)
|
|
138
|
-
|
|
139
|
-
default_factory=lambda: defaultdict(lambda: defaultdict(
|
|
142
|
+
samples: Dict[str, Dict[str, List[str]]] = field(
|
|
143
|
+
default_factory=lambda: defaultdict(lambda: defaultdict(list))
|
|
140
144
|
)
|
|
141
145
|
_file_based_dict: Optional[FileBackedDict[SourceReportSubtypes]] = None
|
|
142
146
|
|
|
147
|
+
# We are adding this to make querying easier for fine-grained lineage
|
|
148
|
+
_fine_grained_lineage_special_case_name = "fineGrainedLineages"
|
|
149
|
+
_samples_to_add: int = 20
|
|
150
|
+
_lineage_aspects_seen: Set[str] = field(default_factory=set)
|
|
151
|
+
|
|
143
152
|
def __post_init__(self) -> None:
|
|
144
153
|
self._file_based_dict = FileBackedDict(
|
|
145
154
|
tablename="urn_aspects",
|
|
@@ -157,6 +166,151 @@ class ExamplesReport(Report, Closeable):
|
|
|
157
166
|
self._file_based_dict.close()
|
|
158
167
|
self._file_based_dict = None
|
|
159
168
|
|
|
169
|
+
def _build_aspects_where_clause(self, aspects: List[str]) -> str:
|
|
170
|
+
"""Build WHERE clause for matching any of the given aspects."""
|
|
171
|
+
if not aspects:
|
|
172
|
+
return ""
|
|
173
|
+
|
|
174
|
+
conditions = []
|
|
175
|
+
for aspect in aspects:
|
|
176
|
+
conditions.append(f"aspects LIKE '%{aspect}%'")
|
|
177
|
+
|
|
178
|
+
return " OR ".join(conditions)
|
|
179
|
+
|
|
180
|
+
def _collect_samples_by_subtype(self, where_clause: str, sample_key: str) -> None:
|
|
181
|
+
"""Helper method to collect samples organized by subtype for a given where clause."""
|
|
182
|
+
|
|
183
|
+
subtype_query = f"""
|
|
184
|
+
SELECT DISTINCT subTypes
|
|
185
|
+
FROM urn_aspects
|
|
186
|
+
WHERE {where_clause}
|
|
187
|
+
"""
|
|
188
|
+
assert self._file_based_dict is not None
|
|
189
|
+
subtypes = set()
|
|
190
|
+
for row in self._file_based_dict.sql_query(subtype_query):
|
|
191
|
+
sub_type = row["subTypes"] or "unknown"
|
|
192
|
+
subtypes.add(sub_type)
|
|
193
|
+
|
|
194
|
+
for sub_type in subtypes:
|
|
195
|
+
query = f"""
|
|
196
|
+
SELECT urn
|
|
197
|
+
FROM urn_aspects
|
|
198
|
+
WHERE {where_clause} AND subTypes = ?
|
|
199
|
+
limit {self._samples_to_add}
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
for row in self._file_based_dict.sql_query(query, (sub_type,)):
|
|
203
|
+
self.samples[sample_key][sub_type].append(row["urn"])
|
|
204
|
+
|
|
205
|
+
def _collect_samples_by_aspects(self, aspects: List[str], sample_key: str) -> None:
|
|
206
|
+
"""Helper method to collect samples for entities that have any of the given aspects."""
|
|
207
|
+
if not aspects:
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
where_clause = self._build_aspects_where_clause(aspects)
|
|
211
|
+
self._collect_samples_by_subtype(where_clause, sample_key)
|
|
212
|
+
|
|
213
|
+
def _collect_samples_by_lineage_aspects(
|
|
214
|
+
self, aspects: List[str], sample_key: str
|
|
215
|
+
) -> None:
|
|
216
|
+
"""Helper method to collect samples for entities that have any of the given lineage aspects.
|
|
217
|
+
|
|
218
|
+
Lineage aspects are stored in JSON format and require quote escaping in LIKE clauses.
|
|
219
|
+
"""
|
|
220
|
+
if not aspects:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
lineage_conditions = []
|
|
224
|
+
for aspect in aspects:
|
|
225
|
+
lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
|
|
226
|
+
|
|
227
|
+
where_clause = " OR ".join(lineage_conditions)
|
|
228
|
+
self._collect_samples_by_subtype(where_clause, sample_key)
|
|
229
|
+
|
|
230
|
+
def _collect_samples_with_all_conditions(self, sample_key: str) -> None:
|
|
231
|
+
"""
|
|
232
|
+
Collect samples for entities that have lineage, profiling, and usage aspects.
|
|
233
|
+
These specific 3 cases are added here as these URNs will be shown in the UI. Subject to change in future.
|
|
234
|
+
"""
|
|
235
|
+
if not self._lineage_aspects_seen:
|
|
236
|
+
return
|
|
237
|
+
assert self._file_based_dict is not None
|
|
238
|
+
|
|
239
|
+
# Build lineage conditions using the same logic as _collect_samples_by_lineage_aspects
|
|
240
|
+
lineage_conditions = []
|
|
241
|
+
for aspect in self._lineage_aspects_seen:
|
|
242
|
+
lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
|
|
243
|
+
lineage_where_clause = " OR ".join(lineage_conditions)
|
|
244
|
+
|
|
245
|
+
# Build profiling conditions using the same logic as _collect_samples_by_aspects
|
|
246
|
+
profiling_where_clause = self._build_aspects_where_clause(["datasetProfile"])
|
|
247
|
+
|
|
248
|
+
# Build usage conditions using the same logic as _collect_samples_by_aspects
|
|
249
|
+
usage_where_clause = self._build_aspects_where_clause(
|
|
250
|
+
[
|
|
251
|
+
"datasetUsageStatistics",
|
|
252
|
+
"chartUsageStatistics",
|
|
253
|
+
"dashboardUsageStatistics",
|
|
254
|
+
]
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
query = f"""
|
|
258
|
+
SELECT urn, subTypes
|
|
259
|
+
FROM urn_aspects
|
|
260
|
+
WHERE ({lineage_where_clause})
|
|
261
|
+
AND ({profiling_where_clause})
|
|
262
|
+
AND ({usage_where_clause})
|
|
263
|
+
limit {self._samples_to_add}
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
for row in self._file_based_dict.sql_query(query):
|
|
267
|
+
sub_type = row["subTypes"] or "unknown"
|
|
268
|
+
self.samples[sample_key][sub_type].append(row["urn"])
|
|
269
|
+
|
|
270
|
+
def _has_fine_grained_lineage(
|
|
271
|
+
self, mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper]
|
|
272
|
+
) -> bool:
|
|
273
|
+
if isinstance(mcp.aspect, UpstreamLineageClass):
|
|
274
|
+
upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
|
|
275
|
+
if upstream_lineage.fineGrainedLineages:
|
|
276
|
+
return True
|
|
277
|
+
return False
|
|
278
|
+
|
|
279
|
+
def _update_file_based_dict(
|
|
280
|
+
self,
|
|
281
|
+
urn: str,
|
|
282
|
+
entityType: str,
|
|
283
|
+
aspectName: str,
|
|
284
|
+
mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper],
|
|
285
|
+
) -> None:
|
|
286
|
+
if is_lineage_aspect(entityType, aspectName):
|
|
287
|
+
self._lineage_aspects_seen.add(aspectName)
|
|
288
|
+
has_fine_grained_lineage = self._has_fine_grained_lineage(mcp)
|
|
289
|
+
|
|
290
|
+
sub_type = "unknown"
|
|
291
|
+
if isinstance(mcp.aspect, SubTypesClass):
|
|
292
|
+
sub_type = mcp.aspect.typeNames[0]
|
|
293
|
+
|
|
294
|
+
assert self._file_based_dict is not None
|
|
295
|
+
if urn in self._file_based_dict:
|
|
296
|
+
if sub_type != "unknown":
|
|
297
|
+
self._file_based_dict[urn].subType = sub_type
|
|
298
|
+
self._file_based_dict[urn].aspects.add(aspectName)
|
|
299
|
+
if has_fine_grained_lineage:
|
|
300
|
+
self._file_based_dict[urn].aspects.add(
|
|
301
|
+
self._fine_grained_lineage_special_case_name
|
|
302
|
+
)
|
|
303
|
+
self._file_based_dict.mark_dirty(urn)
|
|
304
|
+
else:
|
|
305
|
+
self._file_based_dict[urn] = SourceReportSubtypes(
|
|
306
|
+
urn=urn,
|
|
307
|
+
entity_type=entityType,
|
|
308
|
+
subType=sub_type,
|
|
309
|
+
aspects={aspectName}
|
|
310
|
+
if not has_fine_grained_lineage
|
|
311
|
+
else {aspectName, self._fine_grained_lineage_special_case_name},
|
|
312
|
+
)
|
|
313
|
+
|
|
160
314
|
def _store_workunit_data(self, wu: MetadataWorkUnit) -> None:
|
|
161
315
|
urn = wu.get_urn()
|
|
162
316
|
|
|
@@ -169,41 +323,15 @@ class ExamplesReport(Report, Closeable):
|
|
|
169
323
|
entityType = mcp.entityType
|
|
170
324
|
aspectName = mcp.aspectName
|
|
171
325
|
|
|
172
|
-
if urn not in self._urns_seen:
|
|
173
|
-
self._urns_seen.add(urn)
|
|
174
|
-
self.entities[entityType].append(urn)
|
|
175
|
-
|
|
176
326
|
if aspectName is None:
|
|
177
327
|
continue
|
|
178
|
-
|
|
179
|
-
self.
|
|
180
|
-
sub_type = "unknown"
|
|
181
|
-
if isinstance(mcp.aspect, UpstreamLineageClass):
|
|
182
|
-
upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
|
|
183
|
-
if upstream_lineage.fineGrainedLineages:
|
|
184
|
-
self.aspect_urn_samples[entityType]["fineGrainedLineages"].append(
|
|
185
|
-
urn
|
|
186
|
-
)
|
|
187
|
-
self.aspects[entityType]["fineGrainedLineages"] += 1
|
|
188
|
-
elif isinstance(mcp.aspect, SubTypesClass):
|
|
189
|
-
sub_type = mcp.aspect.typeNames[0]
|
|
190
|
-
assert self._file_based_dict is not None
|
|
191
|
-
if urn in self._file_based_dict:
|
|
192
|
-
if sub_type != "unknown":
|
|
193
|
-
self._file_based_dict[urn].subType = sub_type
|
|
194
|
-
self._file_based_dict[urn].aspects.add(aspectName)
|
|
195
|
-
self._file_based_dict.mark_dirty(urn)
|
|
196
|
-
else:
|
|
197
|
-
self._file_based_dict[urn] = SourceReportSubtypes(
|
|
198
|
-
urn=urn,
|
|
199
|
-
entity_type=entityType,
|
|
200
|
-
subType=sub_type,
|
|
201
|
-
aspects={aspectName},
|
|
202
|
-
)
|
|
328
|
+
|
|
329
|
+
self._update_file_based_dict(urn, entityType, aspectName, mcp)
|
|
203
330
|
|
|
204
331
|
def compute_stats(self) -> None:
|
|
205
332
|
if self._file_based_dict is None:
|
|
206
333
|
return
|
|
334
|
+
|
|
207
335
|
query = """
|
|
208
336
|
SELECT entityType, subTypes, aspects, count(*) as count
|
|
209
337
|
FROM urn_aspects
|
|
@@ -223,11 +351,31 @@ class ExamplesReport(Report, Closeable):
|
|
|
223
351
|
for aspect in aspects:
|
|
224
352
|
entity_subtype_aspect_counts[entity_type][sub_type][aspect] += count
|
|
225
353
|
|
|
354
|
+
self.aspects.clear()
|
|
226
355
|
self.aspects_by_subtypes.clear()
|
|
356
|
+
_aspects_seen: Set[str] = set()
|
|
227
357
|
for entity_type, subtype_counts in entity_subtype_aspect_counts.items():
|
|
228
358
|
for sub_type, aspect_counts in subtype_counts.items():
|
|
359
|
+
for aspect, count in aspect_counts.items():
|
|
360
|
+
self.aspects[entity_type][aspect] += count
|
|
361
|
+
_aspects_seen.add(aspect)
|
|
229
362
|
self.aspects_by_subtypes[entity_type][sub_type] = dict(aspect_counts)
|
|
230
363
|
|
|
364
|
+
self.samples.clear()
|
|
365
|
+
self._collect_samples_by_aspects(["datasetProfile"], "profiling")
|
|
366
|
+
self._collect_samples_by_aspects(
|
|
367
|
+
[
|
|
368
|
+
"datasetUsageStatistics",
|
|
369
|
+
"chartUsageStatistics",
|
|
370
|
+
"dashboardUsageStatistics",
|
|
371
|
+
],
|
|
372
|
+
"usage",
|
|
373
|
+
)
|
|
374
|
+
self._collect_samples_by_lineage_aspects(
|
|
375
|
+
list(self._lineage_aspects_seen), "lineage"
|
|
376
|
+
)
|
|
377
|
+
self._collect_samples_with_all_conditions("all_3")
|
|
378
|
+
|
|
231
379
|
|
|
232
380
|
class EntityFilterReport(ReportAttribute):
|
|
233
381
|
type: str
|