acryl-datahub 1.1.0.5rc6__py3-none-any.whl → 1.1.0.5rc7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (38) hide show
  1. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/METADATA +2547 -2547
  2. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/RECORD +38 -35
  3. datahub/_version.py +1 -1
  4. datahub/ingestion/api/report.py +183 -35
  5. datahub/ingestion/autogenerated/capability_summary.json +3366 -0
  6. datahub/ingestion/autogenerated/lineage.json +401 -0
  7. datahub/ingestion/autogenerated/lineage_helper.py +30 -128
  8. datahub/ingestion/run/pipeline.py +4 -1
  9. datahub/ingestion/source/bigquery_v2/bigquery.py +23 -22
  10. datahub/ingestion/source/cassandra/cassandra_profiling.py +6 -5
  11. datahub/ingestion/source/common/subtypes.py +1 -1
  12. datahub/ingestion/source/data_lake_common/object_store.py +40 -0
  13. datahub/ingestion/source/dremio/dremio_source.py +6 -3
  14. datahub/ingestion/source/gcs/gcs_source.py +4 -1
  15. datahub/ingestion/source/ge_data_profiler.py +28 -20
  16. datahub/ingestion/source/kafka_connect/source_connectors.py +59 -4
  17. datahub/ingestion/source/mock_data/datahub_mock_data.py +45 -0
  18. datahub/ingestion/source/redshift/usage.py +4 -3
  19. datahub/ingestion/source/s3/source.py +19 -3
  20. datahub/ingestion/source/snowflake/snowflake_queries.py +47 -3
  21. datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
  22. datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
  23. datahub/ingestion/source/unity/proxy.py +4 -3
  24. datahub/ingestion/source/unity/source.py +10 -8
  25. datahub/integrations/assertion/snowflake/compiler.py +4 -3
  26. datahub/metadata/_internal_schema_classes.py +85 -4
  27. datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +2 -0
  28. datahub/metadata/schema.avsc +54 -1
  29. datahub/metadata/schemas/CorpUserSettings.avsc +17 -1
  30. datahub/metadata/schemas/GlobalSettingsInfo.avsc +37 -0
  31. datahub/sdk/lineage_client.py +2 -0
  32. datahub/sql_parsing/sql_parsing_aggregator.py +3 -3
  33. datahub/sql_parsing/sqlglot_lineage.py +2 -0
  34. datahub/utilities/sqlalchemy_query_combiner.py +5 -2
  35. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/WHEEL +0 -0
  36. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/entry_points.txt +0 -0
  37. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/licenses/LICENSE +0 -0
  38. {acryl_datahub-1.1.0.5rc6.dist-info → acryl_datahub-1.1.0.5rc7.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
- acryl_datahub-1.1.0.5rc6.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
1
+ acryl_datahub-1.1.0.5rc7.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
2
2
  datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
3
3
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
4
- datahub/_version.py,sha256=P6EdFT0jTjQYZMQLbjiR8hZ8LYv0hNCOD3SH085iCVw,323
4
+ datahub/_version.py,sha256=sUspZGha0FX624vViKEbY5GTEpcVIfY_B0Gq4roHDZ0,323
5
5
  datahub/entrypoints.py,sha256=H-YFTvxTJOgpWsFBVlxyb1opjkq-hjTzNmjy5Fq3RHg,8992
6
6
  datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
7
7
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -148,7 +148,7 @@ datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPs
148
148
  datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
149
149
  datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
150
150
  datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
151
- datahub/ingestion/api/report.py,sha256=Ex3c0L0Rl67kRNDkgxt3EEEEPXKbEdcvvTPf_KomLMU,9512
151
+ datahub/ingestion/api/report.py,sha256=SRZ1DNyHUV08uxA3qGzWOX-lVfSSwh4ln6NOSwMYhIA,14807
152
152
  datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
153
153
  datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
154
154
  datahub/ingestion/api/source.py,sha256=WcpmuWRua87_UbaSY8GtkLkC0Gql0pM_i5FJN5ALh88,19669
@@ -159,7 +159,9 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
159
159
  datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
160
160
  datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=fMjPnyWEofIZV52E2AFYU3IgBJwyZvbygXxCJyEtcWI,4442
161
161
  datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
162
- datahub/ingestion/autogenerated/lineage_helper.py,sha256=3PQKUcWkkgIopWw8dMZIkTHhpeJqPO6A4k5oyu3XfZM,6091
162
+ datahub/ingestion/autogenerated/capability_summary.json,sha256=CiZ0OZlU40jaItCrOYwdUfnb2JVZSb-KYBr2-MiMfpc,104572
163
+ datahub/ingestion/autogenerated/lineage.json,sha256=wjaXxrzpo3M9ppc80TKIgCTuCg0jpb-D9MhBsHXaMOY,10004
164
+ datahub/ingestion/autogenerated/lineage_helper.py,sha256=MJrHGuDvM_9hrFfRt0dkzE58j1xIsvgW6UIQBEKqdYE,2701
163
165
  datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
164
166
  datahub/ingestion/extractor/extractor_registry.py,sha256=f7CLfW3pr29QZkXSHbp7HjUrsdw7ejQJmot-tiSPcqc,342
165
167
  datahub/ingestion/extractor/json_ref_patch.py,sha256=4g3ZWHn7rwS74jUvSXJiGpi-UKHhiSYKKgBeU4E5ukE,1448
@@ -191,7 +193,7 @@ datahub/ingestion/reporting/file_reporter.py,sha256=tiWukmMxHrTQI3rOAumsq6lRlw8T
191
193
  datahub/ingestion/reporting/reporting_provider_registry.py,sha256=jTYSh3T4sensjnHQfPLiIcbA2dG8w0px9ghChAJjGdU,310
192
194
  datahub/ingestion/run/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
193
195
  datahub/ingestion/run/connection.py,sha256=mngNzr5aRLUDa5Izqxa0xkdDEqEqcDuacWSKIlkdvPc,1483
194
- datahub/ingestion/run/pipeline.py,sha256=lMUQmxC1z2YPYTlTBzrOMGh7QMzqb7jgFoWf8tAbmjU,30250
196
+ datahub/ingestion/run/pipeline.py,sha256=FGRvCo9aRIMtDa4a3JHy1u2xi70p9TQuiiC17yXZVSc,30287
195
197
  datahub/ingestion/run/pipeline_config.py,sha256=joG1j9OlwJhb8zqv4TY6_FSzOaKOx6xsBu255A5lP8g,4101
196
198
  datahub/ingestion/run/sink_callback.py,sha256=xZAzaKkR0dcQP838pMJWsA52oaQXV5BiqXkpxEpJ_9U,2856
197
199
  datahub/ingestion/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -209,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
209
211
  datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
210
212
  datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
211
213
  datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
212
- datahub/ingestion/source/ge_data_profiler.py,sha256=dvwTLK95xx1vuLPzigredqXiv0nyZVKas1dP7zcy3jU,67807
214
+ datahub/ingestion/source/ge_data_profiler.py,sha256=TnAupfORYFzcgP7akQEqeuvy1qO0sxMetb0B75f95jM,67980
213
215
  datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
214
216
  datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
215
217
  datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
@@ -254,7 +256,7 @@ datahub/ingestion/source/azure/abs_folder_utils.py,sha256=7skXus-4fSIoKpqCeU-GG0
254
256
  datahub/ingestion/source/azure/abs_utils.py,sha256=KdAlCK-PMrn35kFHxz5vrsjajyx2PD5GRgoBKdoRvcg,2075
255
257
  datahub/ingestion/source/azure/azure_common.py,sha256=Zl0pPuE6L3QcM5B1P0LsPthZmD0h7fUUS0kg2okl6IY,4053
256
258
  datahub/ingestion/source/bigquery_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
257
- datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=dslbjihZMg9Utt8V8DYIucqQfychl_MB-gaDTmsMqe0,15005
259
+ datahub/ingestion/source/bigquery_v2/bigquery.py,sha256=QNG_3IU6xFyGz7Xq2dHyPS0C-LCuJyMAFpqPrMLRBsE,15100
258
260
  datahub/ingestion/source/bigquery_v2/bigquery_audit.py,sha256=kEwWhq3ch6WT4q4hcX8-fvQh28KgrNfspFwIytO3vQA,25103
259
261
  datahub/ingestion/source/bigquery_v2/bigquery_audit_log_api.py,sha256=LuGJ6LgPViLIfDQfylxlQ3CA7fZYM5MDt8M-7sfzm84,5096
260
262
  datahub/ingestion/source/bigquery_v2/bigquery_config.py,sha256=bG1soRawCLhJw_15L2fJmFfj1kntTthV6ng4LZOnwko,21916
@@ -277,17 +279,17 @@ datahub/ingestion/source/cassandra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
277
279
  datahub/ingestion/source/cassandra/cassandra.py,sha256=k2EKpeUUZ0O_RwQ6NXM15vnTsY3OgST4e5LG8Eg6Coo,14534
278
280
  datahub/ingestion/source/cassandra/cassandra_api.py,sha256=b7MApc3_tEfHoj-6ub6snkcv_DweL1wi_TGJjAA1-yU,13516
279
281
  datahub/ingestion/source/cassandra/cassandra_config.py,sha256=Ga9915cDZukR5-u2tMNx5Jkf8eza2oAE5YS_sQIVEVQ,4222
280
- datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=ZqsAY8NFsrrLqOduV7Aem2eJLtc2_OU9tW4tc_dh0V8,10984
282
+ datahub/ingestion/source/cassandra/cassandra_profiling.py,sha256=FdgPK_8s8otTOJDqNM4rpF6Mn4lFWbnjTaKEChzn2iE,11011
281
283
  datahub/ingestion/source/cassandra/cassandra_utils.py,sha256=j-LidYkaCTmGnpUVNLsax_c3z32PsQbsbHeYojygd1s,5105
282
284
  datahub/ingestion/source/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
283
285
  datahub/ingestion/source/common/data_platforms.py,sha256=HhuP3YIEi2WpyKDjUU8RiM0a2qjHWQcvc8kcqub0cVo,548
284
286
  datahub/ingestion/source/common/data_reader.py,sha256=XbSxiRTYrk6seOz0ZjVjzSpGvP8lEjmqXrNI4cdYYmQ,1819
285
287
  datahub/ingestion/source/common/gcp_credentials_config.py,sha256=_NapGkAqZMbXNClLlmOfANS7U6rChhdthRX9s9iUv9k,2411
286
- datahub/ingestion/source/common/subtypes.py,sha256=SvsaT2PM5VG7iH3c0iyvAK3oyWS_EpB3LhFOoX-e5Ws,4493
288
+ datahub/ingestion/source/common/subtypes.py,sha256=-z-QseXQC-Pt8pbbw4CsMRsqIcNzZ-mz-9iQSXyU65s,4493
287
289
  datahub/ingestion/source/data_lake_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
288
290
  datahub/ingestion/source/data_lake_common/config.py,sha256=qUk83B01hjuBKHvVz8SmXnVCy5eFj-2-2QLEOrAdbgk,359
289
291
  datahub/ingestion/source/data_lake_common/data_lake_utils.py,sha256=55mK0nsehqGDTUOol9Oi3jZs8Pb04PIsHdC2WPP0dkg,6576
290
- datahub/ingestion/source/data_lake_common/object_store.py,sha256=T7onzor_15MR-7ecxqcd6YnGY0-bGXHJzseg6nfo2Og,20302
292
+ datahub/ingestion/source/data_lake_common/object_store.py,sha256=NkbCXq56sdGkyBAxcrpSVE6GySLrh9E2jYZU6lYPupM,21644
291
293
  datahub/ingestion/source/data_lake_common/path_spec.py,sha256=ekJAr4-PE2RhzQnmKb2xcSs_YncC1Dz95-UrXI67Vos,23584
292
294
  datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
293
295
  datahub/ingestion/source/datahub/config.py,sha256=bjR1U3F85FbtgqmLDW-f_4dQvuw5AsJQxdQlOUeHDUk,5126
@@ -317,7 +319,7 @@ datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py,sha256=MQk8BAHL
317
319
  datahub/ingestion/source/dremio/dremio_entities.py,sha256=1gZrNqTp3Pm6vqGDQaWt3HkxEuHKxpGYQ4geVoFvxWI,15147
318
320
  datahub/ingestion/source/dremio/dremio_profiling.py,sha256=TAcnpo8ZRKhLDHnQSJzJg3YdwTSyEa73LUAzENs7wG4,12287
319
321
  datahub/ingestion/source/dremio/dremio_reporting.py,sha256=YRKM6PvoJYHLBXmOGwkgou_8x8_oA2xaqTWWoVuwFMY,2247
320
- datahub/ingestion/source/dremio/dremio_source.py,sha256=baUW3f6Y7WWbHXo9GqmBzZqXilMo1MbG3hvDS-bwthI,25164
322
+ datahub/ingestion/source/dremio/dremio_source.py,sha256=ODIUhDr6Nx5ziV7Buo-ayT40WxkSKNhFhOlM6fmw-xM,25236
321
323
  datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=wA1hqKk9cKMJDyEdZRQcDDLZPGYwuNqrvleUHTkWgrQ,10508
322
324
  datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
323
325
  datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
@@ -334,7 +336,7 @@ datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=mUWcMt-_FL1SYGIgI4lGZD
334
336
  datahub/ingestion/source/gc/execution_request_cleanup.py,sha256=y-9ZIs_DZPUzYH1CI6HmaAZg3olNNA7MjT8HrCqAI0k,11159
335
337
  datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py,sha256=4-qQR_2HGIYU8kC2hRIsJyKKMb9lKq4B6paJm_abUk4,12628
336
338
  datahub/ingestion/source/gcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
337
- datahub/ingestion/source/gcs/gcs_source.py,sha256=ZdEmfy9TPd8nR8zlTUBvWDb9_cNFxksp-34O53L_ATE,6798
339
+ datahub/ingestion/source/gcs/gcs_source.py,sha256=mrecg2ewT2ATrUuEYg3cMLPXXAWwc2WJzJFmAHNWuTc,7063
338
340
  datahub/ingestion/source/gcs/gcs_utils.py,sha256=Kd2usZYIMFeSuE6_tJ4OoHGOdvG8mWaScFuAcIkC6P0,1789
339
341
  datahub/ingestion/source/git/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
340
342
  datahub/ingestion/source/git/git_import.py,sha256=5CT6vMDb0MDctCtShnxb3JVihULtvkYGr9judHJFsOk,4143
@@ -362,7 +364,7 @@ datahub/ingestion/source/kafka_connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
362
364
  datahub/ingestion/source/kafka_connect/common.py,sha256=lH64n1v_rJamWGfidBeuQJj8W1_IvOBpXQLR2YZaEvQ,7057
363
365
  datahub/ingestion/source/kafka_connect/kafka_connect.py,sha256=AVAgBvgH7kM9I2ke3mwr8CfIL1J2SdVHH_86rnCFwrM,17727
364
366
  datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfICTXIsv3ZfzlrMA4VlZXKQTcFCyIAU,17220
365
- datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=OQ0vjz9xF0T30pRln_gDvelmaOE5jTAxwsCtm1K4SWM,21080
367
+ datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
366
368
  datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
367
369
  datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
368
370
  datahub/ingestion/source/looker/looker_common.py,sha256=wwCRHyd0F799CEBboryQ4TZV057krnY2nMJOkUWc268,64765
@@ -389,7 +391,7 @@ datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
389
391
  datahub/ingestion/source/metadata/business_glossary.py,sha256=T_RJHst6iQRghJNmLLPeSBMEDsbEKf3yBldOAgMcGuo,19666
390
392
  datahub/ingestion/source/metadata/lineage.py,sha256=PA4JwSeQ-30XFMN4O5tPwIu-hZF1e-xMZ_CnEUE2c-Q,9595
391
393
  datahub/ingestion/source/mock_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
392
- datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=8ibA8jGOhz6SGDgvosj0ovI7aR3eKRbYqCMIba-wAt8,14460
394
+ datahub/ingestion/source/mock_data/datahub_mock_data.py,sha256=PLQU1WnxDljZJGifoh6CqIQdBZrQAeBxtV-9NLdxM8o,15847
393
395
  datahub/ingestion/source/mock_data/datahub_mock_data_report.py,sha256=sV_H7JgcuVbrpIBqtGse_BBigMdqP32ZXuanpeXmwVI,331
394
396
  datahub/ingestion/source/mock_data/table_naming_helper.py,sha256=oIC1vcOx76Vl63O9kcjP_iInBHyS-ATdN3Y932TCCZg,3283
395
397
  datahub/ingestion/source/neo4j/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -438,13 +440,13 @@ datahub/ingestion/source/redshift/redshift.py,sha256=p6rOOCjxNnPpTn-vFjgISMMjtUT
438
440
  datahub/ingestion/source/redshift/redshift_data_reader.py,sha256=zc69jwXHdF-w8J4Hq-ZQ6BjHQ75Ij2iNDMpoRJlcmlU,1724
439
441
  datahub/ingestion/source/redshift/redshift_schema.py,sha256=7F-l_omOuKMuGE_rBWXVPG_GWXFKnCMzC4frNxZB9cs,24800
440
442
  datahub/ingestion/source/redshift/report.py,sha256=O3QFozHlmMbH9b7KxbqhgTgr_0tCryj6FIzMiN6kRxw,3044
441
- datahub/ingestion/source/redshift/usage.py,sha256=KGarj9QY1t2Q92-o_xUsYWqt2iuLqRrVlno2hW8F3Mo,17454
443
+ datahub/ingestion/source/redshift/usage.py,sha256=Q7R-caJovLXv33uZepMGX5Cvm4DqQSLZdiL_s-p06wU,17473
442
444
  datahub/ingestion/source/s3/__init__.py,sha256=HjqFPj11WtNFZM3kcVshlDb7kOsc19-l_3LM8PBjlJM,56
443
445
  datahub/ingestion/source/s3/config.py,sha256=lElFXgEpKDT9SVoiXvtx98wV6Gp880qP4pLQaOGJGOo,7828
444
446
  datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
445
447
  datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
446
448
  datahub/ingestion/source/s3/report.py,sha256=9Ej1UCChw963UpGw1-7asi5vFrOM232gfgG8bRdKPp0,667
447
- datahub/ingestion/source/s3/source.py,sha256=ISN4OoLEXRx54pD3seoUUQbPOV54uIrC-QJx5n-CiBI,60246
449
+ datahub/ingestion/source/s3/source.py,sha256=DuJ5G0S2ukZFRL-aO8b4ihzhTlxvbzbj2_GyS8zLb7s,60968
448
450
  datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
449
451
  datahub/ingestion/source/sac/sac.py,sha256=0s_JxHGOhit3Wvgbg7qQi-Z9j9_TgBX_I1yOR3L6-rA,30243
450
452
  datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
@@ -475,7 +477,7 @@ datahub/ingestion/source/snowflake/snowflake_connection.py,sha256=3-nP3HHCblUnUH
475
477
  datahub/ingestion/source/snowflake/snowflake_data_reader.py,sha256=ffR5E2uhD71FUMXd3XOg2rHwrp1rbbGEFTAbqKcmI2s,2195
476
478
  datahub/ingestion/source/snowflake/snowflake_lineage_v2.py,sha256=eWUlWMc5X2X_Y1I2peworFWLLsXQjryEHxPDuSqrowg,21683
477
479
  datahub/ingestion/source/snowflake/snowflake_profiler.py,sha256=PmQi-qDlRhdJ-PsJ7x-EScIiswWRAxDDOKHydvN3mTY,7404
478
- datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=Jc5H5Oo8JmSNV8MXCnyq_UH2XcYcLg2S1vJdQNkvuy4,31189
480
+ datahub/ingestion/source/snowflake/snowflake_queries.py,sha256=PHDFks4rhSfqsPLcamMZJDDJic8988P3WNWglXIUN6M,33017
479
481
  datahub/ingestion/source/snowflake/snowflake_query.py,sha256=kqOxibplHyDhdioue8MeXBzeDS9d274-dspb-wyWMMI,38165
480
482
  datahub/ingestion/source/snowflake/snowflake_report.py,sha256=O-465aBA8uaYZ6WepP7i6cgK6Q1jXJPjDA1j9C8klus,6762
481
483
  datahub/ingestion/source/snowflake/snowflake_schema.py,sha256=1yGBbs2aWIdHnrwgeTR7J2lqxbbBsIt8ejCLumIpLEA,27274
@@ -483,9 +485,10 @@ datahub/ingestion/source/snowflake/snowflake_schema_gen.py,sha256=_37-AQyI4uGt4f
483
485
  datahub/ingestion/source/snowflake/snowflake_shares.py,sha256=maZyFkfrbVogEFM0tTKRiNp9c_1muv6YfleSd3q0umI,6341
484
486
  datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=WJfsP8w3HceUkM6GKONtWCTtYvTdR209cRW9g66xyYE,5671
485
487
  datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=eA9xh-G1Ydr1OwUUtrbXUWp26hE1jF0zvyKNky_i_nQ,8887
486
- datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
488
+ datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=mM0v9b4PHRJAT-SdRids3wdzc5O96gWCCww3e42itV8,24982
487
489
  datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=w-SftpjHSOf_6S2T2srHJyJta3MHS0usmn4Z9jgx4QE,13858
488
490
  datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=eqOVRYmFiWQPAIDWUcOo5QZkG05xanlEbxz4aDTvSNM,34706
491
+ datahub/ingestion/source/snowflake/stored_proc_lineage.py,sha256=XVEP8rGGIqds8f7RRoUHGajsT0myBpYS_u5aVdwggBU,5308
489
492
  datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
490
493
  datahub/ingestion/source/sql/athena.py,sha256=lQ_zniHtmtSFCnSmzM2dG7ND6zOZmhu6t-I4ZOeQKho,27652
491
494
  datahub/ingestion/source/sql/athena_properties_extractor.py,sha256=n2SvqeUbNWxiWWdkDs8VYlUPlLwfZzZy9AIa-V4D7AY,28531
@@ -547,11 +550,11 @@ datahub/ingestion/source/unity/config.py,sha256=7QosoBthg9kirHfXev_vhefkobUxYnp1
547
550
  datahub/ingestion/source/unity/connection_test.py,sha256=B143Wb28fS0V4GhygU9hzKqiArWBjsQO54IUCPf23dc,2586
548
551
  datahub/ingestion/source/unity/ge_profiler.py,sha256=NBRHZceq-f95iUn7u0h7cgcd9nAc48Aa-lmp_BqE0As,8409
549
552
  datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
550
- datahub/ingestion/source/unity/proxy.py,sha256=E8ZhWIY3j7gjEz8ttWOeHlom0jCMJXkWHVBpK5p0RLY,22768
553
+ datahub/ingestion/source/unity/proxy.py,sha256=i15xEfjt7sbajbDIA9cbrxKWLByoL2drI8lnvWPceqc,22787
551
554
  datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
552
555
  datahub/ingestion/source/unity/proxy_types.py,sha256=qrvHiwPzl5cPX-KRvcIGGeJVdr0I8XUQmoAI6ErZ-v8,9371
553
556
  datahub/ingestion/source/unity/report.py,sha256=XFT9oQfvEB4RkTvWGgFOoQuLPUN_AIoPXZ79xeDhGHQ,2831
554
- datahub/ingestion/source/unity/source.py,sha256=cMGnbzLBoDEiS6atNz8APGMS1i3s62ZS1vqKgooUDsk,49726
557
+ datahub/ingestion/source/unity/source.py,sha256=_cqxiQYZWyNXHpnpZ6EGO9WRfL6a_7fwLwv5cVZdto4,49826
555
558
  datahub/ingestion/source/unity/tag_entities.py,sha256=iWl6nRAWSye1hoFDx_Xh4aT53PN0sGzlX7n1-oTVUv8,11568
556
559
  datahub/ingestion/source/unity/usage.py,sha256=0wETBAaZvHI_EGgBlxX3bKsVHEAdnUV8_bKI_lbyWjY,11500
557
560
  datahub/ingestion/source/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -599,7 +602,7 @@ datahub/integrations/assertion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
599
602
  datahub/integrations/assertion/common.py,sha256=rUkcyJ87vedWqJPVGco0e6FPF07zj8DNOPW1knIvDg8,2297
600
603
  datahub/integrations/assertion/registry.py,sha256=mmeYpQREFVrLWpLcA0qYZtbrMX4vAGcDoQ59wtMgEcg,307
601
604
  datahub/integrations/assertion/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
602
- datahub/integrations/assertion/snowflake/compiler.py,sha256=8QrgVKqMo6oow93DCDvQtFzo-CNRtVjbwmA-14TU-GQ,9955
605
+ datahub/integrations/assertion/snowflake/compiler.py,sha256=qL757BVcpR9MoUOHPKBmDoid_5OSNbCUbbt3BivEds8,9970
603
606
  datahub/integrations/assertion/snowflake/dmf_generator.py,sha256=ti4EnSlTGdAWmBcxgUqqilCGhLVGutpfSatbENP94O8,746
604
607
  datahub/integrations/assertion/snowflake/field_metric_sql_generator.py,sha256=V2kwDTgg3ChnxW2h5pITpUwtCiwL9Lmhg4p6T5iS9IE,6513
605
608
  datahub/integrations/assertion/snowflake/field_values_metric_sql_generator.py,sha256=aVVOhYPZKVITVNfGEQO54_XjjRuMplsu-yeFLE7MhxA,8950
@@ -615,8 +618,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
615
618
  datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
616
619
  datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
617
620
  datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
618
- datahub/metadata/_internal_schema_classes.py,sha256=E9vXJ4SehOcRTliF8ytVra6TfW41JlEjGxILjMQiLFQ,1040331
619
- datahub/metadata/schema.avsc,sha256=AJC7ZJs37yBAIEhDBK2UPmze2pWWXhYRcf5qlpdW5fc,719162
621
+ datahub/metadata/_internal_schema_classes.py,sha256=h_FmxHQdH1tChgFYOGgULr6RMgiXN9JfFOoMC3TSoEg,1043586
622
+ datahub/metadata/schema.avsc,sha256=zrngtDblYg2sLuu-5t5IJxRq52TexvIIElfoU99wuZw,720852
620
623
  datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
621
624
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
622
625
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -687,7 +690,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schema/__init__.py,sha256=7JpzLs6S_Ey
687
690
  datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWeznycKnHBfPEGcCHXPEz83Iq9ypjNaoSfeQeDU9g,397
688
691
  datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
689
692
  datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
690
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=brzmpkgl_UI_ExT4UZh3WzE3TEXkyopzCmrU54qf6XY,825
693
+ datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=hKpTDXj2YEBja4hk2fZx3G9yzJ7eCFxcGCbbN0NydTk,933
691
694
  datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
692
695
  datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
693
696
  datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
@@ -728,7 +731,7 @@ datahub/metadata/schemas/CorpUserCredentials.avsc,sha256=S7FkV9K_DGxhb4GFYbM5_lP
728
731
  datahub/metadata/schemas/CorpUserEditableInfo.avsc,sha256=6IrqWidbHP7mRryfVlWAQU0JS34THHTM8_aIKWqClUE,3843
729
732
  datahub/metadata/schemas/CorpUserInfo.avsc,sha256=oObOza-5FLjZyCjj0FN4MNV1DodgTwJSV4APduAggjk,3955
730
733
  datahub/metadata/schemas/CorpUserKey.avsc,sha256=01sbbdr8G-ZP1yc2UfY3gR-YN6b7AvDbCbNpZJ-02J4,1025
731
- datahub/metadata/schemas/CorpUserSettings.avsc,sha256=hEJ5nQxAG-4ahUMQEl8NcJVb1kFp0YIdwHPNTWruD8k,6070
734
+ datahub/metadata/schemas/CorpUserSettings.avsc,sha256=ff-rvW3ihwo6s9usvStx0aXTmCNmcR1TjggDxhbCacc,6529
732
735
  datahub/metadata/schemas/CorpUserStatus.avsc,sha256=yqojAXEQ9CjRhY58RPyTUxzmFbHSANGGaMMbqiYZZIE,2538
733
736
  datahub/metadata/schemas/Cost.avsc,sha256=o4kYZSss2uEwJ6gCA9fhBUoyD5xUqcSxz78vkIXXzGQ,1494
734
737
  datahub/metadata/schemas/DashboardInfo.avsc,sha256=li2lSV6R4V-nz6foOi-NYxt_8ShHWfoKRw6M2BG5530,12907
@@ -834,7 +837,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
834
837
  datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
835
838
  datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
836
839
  datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
837
- datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=juyuGFIZ8HuuJU-eGzIW3z3zpdeIR_DIEm-sTkCfhHE,10874
840
+ datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=HKZisxW2HNm6kwJUBaAVol7suJoxSVCjfvWy55HhalQ,11999
838
841
  datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
839
842
  datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
840
843
  datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
@@ -943,7 +946,7 @@ datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
943
946
  datahub/sdk/dataset.py,sha256=2-iD-HqjgFpCBmU3k8O5tkltmwFj4jaumADiX8sexJU,29465
944
947
  datahub/sdk/entity.py,sha256=Q29AbpS58L4gD8ETwoNIwG-ouytz4c0MSSFi6-jLl_4,6742
945
948
  datahub/sdk/entity_client.py,sha256=xHLGLn6oJfxmHLXo5w4-isPFZGcf8yR5IwyC6pvl_I8,8903
946
- datahub/sdk/lineage_client.py,sha256=M4fN4p8YHpM7zoQ7RCCekV8oPgw8QsWxzMAA8mPoy20,33822
949
+ datahub/sdk/lineage_client.py,sha256=xHTX3B1KnVIh6vQLtJHGPMT6BXXru25UE1IK2T7RB28,33914
947
950
  datahub/sdk/main_client.py,sha256=kJRmydyyaOjUdqOljl0w_Cx7BKkViZmnX-1vZBHY4nw,5266
948
951
  datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
949
952
  datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
@@ -977,10 +980,10 @@ datahub/sql_parsing/fingerprint_utils.py,sha256=3hGiexaQXnE7eZLxo-t7hlTyVQz7womb
977
980
  datahub/sql_parsing/query_types.py,sha256=FKjDzszZzsrCfYfm7dgD6T_8865qxWl767fdGyHWBh4,2720
978
981
  datahub/sql_parsing/schema_resolver.py,sha256=ISuingLcQnOJZkNXBkc73uPwYUbbOtERAjgGhJajDiQ,10782
979
982
  datahub/sql_parsing/split_statements.py,sha256=OIQXA9e4k3G9Z1y7rbgdtZhMWt4FPnq41cE8Jkm9cBY,9542
980
- datahub/sql_parsing/sql_parsing_aggregator.py,sha256=Y70tMjCZTHf86g7REwi0A3jSvwQzv5qsfEfaDyiqnhw,71595
983
+ datahub/sql_parsing/sql_parsing_aggregator.py,sha256=CmvBqCkIkeX606-uZv44N9meL8TnRydKpWFia-_CoEQ,71642
981
984
  datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
982
985
  datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
983
- datahub/sql_parsing/sqlglot_lineage.py,sha256=jchSPPYkFtHpyTRTWR5K0YQM6LIgWR5MtyVNQ6zA2Ig,59915
986
+ datahub/sql_parsing/sqlglot_lineage.py,sha256=IeWxkgOp7wP83Id_0NHy0Ra_L-CmjIs0Hm2ul_TWu7c,60003
984
987
  datahub/sql_parsing/sqlglot_utils.py,sha256=TI11oBu1wrGeUuUGBg7hGTr6lTvztahdqiqXNJYRfbQ,14823
985
988
  datahub/sql_parsing/tool_meta_extractor.py,sha256=EV_g7sOchTSUm2p6wluNJqND7-rDYokVTqqFCM7hQ6c,7599
986
989
  datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1035,7 +1038,7 @@ datahub/utilities/sentinels.py,sha256=p299J2YR62K2Mbuy-2lrGH4BnF9dkRqGo59Wub0hYV
1035
1038
  datahub/utilities/serialized_lru_cache.py,sha256=R9enyN5BlatknX4KFq9euYc2yiE1rRMucPbAHg09Goc,3247
1036
1039
  datahub/utilities/server_config_util.py,sha256=hUjPaA96eQrur6UqSAj7c5lPfLvyqcgKLFOrR7XG3EY,9618
1037
1040
  datahub/utilities/sql_formatter.py,sha256=tYXIsKjKmpKh0JXGxeAPrHkUWYd1SwJNLjUZsfQP2h0,1016
1038
- datahub/utilities/sqlalchemy_query_combiner.py,sha256=oxW20uXz8hV1Zb4fLXvTQ7c3LjACBsrF58TR2_-RSps,14982
1041
+ datahub/utilities/sqlalchemy_query_combiner.py,sha256=ZWGPG5mrKuumSSEQMrUSjUjSKTaDXw71KWI7Ytnv2xE,15027
1039
1042
  datahub/utilities/sqlalchemy_type_converter.py,sha256=H4S4xnnyPozDBHFhBh4rjjoXa5novFzYIUBJy2KSrVc,9805
1040
1043
  datahub/utilities/sqllineage_patch.py,sha256=0Buh50bmEqJFg1HFRCknCnePo1cecI4JmGxVhM_jh2g,1976
1041
1044
  datahub/utilities/stats_collections.py,sha256=9QDEk40UxhmQwDS6I63Gp6fcIBqmXVinKl7x2xHCD34,1702
@@ -1088,8 +1091,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
1088
1091
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
1089
1092
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
1090
1093
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
1091
- acryl_datahub-1.1.0.5rc6.dist-info/METADATA,sha256=pK9PlvvTCMioPebxscqz_snMjStbpOTeZEkqOXiSW-s,182347
1092
- acryl_datahub-1.1.0.5rc6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1093
- acryl_datahub-1.1.0.5rc6.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
1094
- acryl_datahub-1.1.0.5rc6.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1095
- acryl_datahub-1.1.0.5rc6.dist-info/RECORD,,
1094
+ acryl_datahub-1.1.0.5rc7.dist-info/METADATA,sha256=OJxr8Zm0_d6AjnEsFM8eYv7CsCoyeKpj0ifhszN5uYc,181944
1095
+ acryl_datahub-1.1.0.5rc7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1096
+ acryl_datahub-1.1.0.5rc7.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
1097
+ acryl_datahub-1.1.0.5rc7.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1098
+ acryl_datahub-1.1.0.5rc7.dist-info/RECORD,,
datahub/_version.py CHANGED
@@ -1,6 +1,6 @@
1
1
  # Published at https://pypi.org/project/acryl-datahub/.
2
2
  __package_name__ = "acryl-datahub"
3
- __version__ = "1.1.0.5rc6"
3
+ __version__ = "1.1.0.5rc7"
4
4
 
5
5
 
6
6
  def is_dev_mode() -> bool:
@@ -6,19 +6,25 @@ from collections import defaultdict
6
6
  from dataclasses import dataclass, field
7
7
  from datetime import datetime, timedelta
8
8
  from enum import Enum
9
- from typing import Any, Dict, Optional, Set, cast, runtime_checkable
9
+ from typing import Any, Dict, List, Optional, Set, Union, cast, runtime_checkable
10
10
 
11
11
  import humanfriendly
12
12
  import pydantic
13
13
  from pydantic import BaseModel
14
14
  from typing_extensions import Literal, Protocol
15
15
 
16
+ from datahub.emitter.mcp import MetadataChangeProposalWrapper
16
17
  from datahub.emitter.mcp_builder import mcps_from_mce
17
18
  from datahub.ingestion.api.closeable import Closeable
18
19
  from datahub.ingestion.api.report_helpers import format_datetime_relative
19
20
  from datahub.ingestion.api.workunit import MetadataWorkUnit
21
+ from datahub.ingestion.autogenerated.lineage_helper import is_lineage_aspect
20
22
  from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
21
- from datahub.metadata.schema_classes import SubTypesClass, UpstreamLineageClass
23
+ from datahub.metadata.schema_classes import (
24
+ MetadataChangeProposalClass,
25
+ SubTypesClass,
26
+ UpstreamLineageClass,
27
+ )
22
28
  from datahub.utilities.file_backed_collections import FileBackedDict
23
29
  from datahub.utilities.lossy_collections import LossyList
24
30
 
@@ -125,8 +131,6 @@ class ReportAttribute(BaseModel):
125
131
 
126
132
  @dataclass
127
133
  class ExamplesReport(Report, Closeable):
128
- _urns_seen: Set[str] = field(default_factory=set)
129
- entities: Dict[str, list] = field(default_factory=lambda: defaultdict(LossyList))
130
134
  aspects: Dict[str, Dict[str, int]] = field(
131
135
  default_factory=lambda: defaultdict(lambda: defaultdict(int))
132
136
  )
@@ -135,11 +139,16 @@ class ExamplesReport(Report, Closeable):
135
139
  lambda: defaultdict(lambda: defaultdict(int))
136
140
  )
137
141
  )
138
- aspect_urn_samples: Dict[str, Dict[str, LossyList[str]]] = field(
139
- default_factory=lambda: defaultdict(lambda: defaultdict(LossyList))
142
+ samples: Dict[str, Dict[str, List[str]]] = field(
143
+ default_factory=lambda: defaultdict(lambda: defaultdict(list))
140
144
  )
141
145
  _file_based_dict: Optional[FileBackedDict[SourceReportSubtypes]] = None
142
146
 
147
+ # We are adding this to make querying easier for fine-grained lineage
148
+ _fine_grained_lineage_special_case_name = "fineGrainedLineages"
149
+ _samples_to_add: int = 20
150
+ _lineage_aspects_seen: Set[str] = field(default_factory=set)
151
+
143
152
  def __post_init__(self) -> None:
144
153
  self._file_based_dict = FileBackedDict(
145
154
  tablename="urn_aspects",
@@ -157,6 +166,151 @@ class ExamplesReport(Report, Closeable):
157
166
  self._file_based_dict.close()
158
167
  self._file_based_dict = None
159
168
 
169
+ def _build_aspects_where_clause(self, aspects: List[str]) -> str:
170
+ """Build WHERE clause for matching any of the given aspects."""
171
+ if not aspects:
172
+ return ""
173
+
174
+ conditions = []
175
+ for aspect in aspects:
176
+ conditions.append(f"aspects LIKE '%{aspect}%'")
177
+
178
+ return " OR ".join(conditions)
179
+
180
+ def _collect_samples_by_subtype(self, where_clause: str, sample_key: str) -> None:
181
+ """Helper method to collect samples organized by subtype for a given where clause."""
182
+
183
+ subtype_query = f"""
184
+ SELECT DISTINCT subTypes
185
+ FROM urn_aspects
186
+ WHERE {where_clause}
187
+ """
188
+ assert self._file_based_dict is not None
189
+ subtypes = set()
190
+ for row in self._file_based_dict.sql_query(subtype_query):
191
+ sub_type = row["subTypes"] or "unknown"
192
+ subtypes.add(sub_type)
193
+
194
+ for sub_type in subtypes:
195
+ query = f"""
196
+ SELECT urn
197
+ FROM urn_aspects
198
+ WHERE {where_clause} AND subTypes = ?
199
+ limit {self._samples_to_add}
200
+ """
201
+
202
+ for row in self._file_based_dict.sql_query(query, (sub_type,)):
203
+ self.samples[sample_key][sub_type].append(row["urn"])
204
+
205
+ def _collect_samples_by_aspects(self, aspects: List[str], sample_key: str) -> None:
206
+ """Helper method to collect samples for entities that have any of the given aspects."""
207
+ if not aspects:
208
+ return
209
+
210
+ where_clause = self._build_aspects_where_clause(aspects)
211
+ self._collect_samples_by_subtype(where_clause, sample_key)
212
+
213
+ def _collect_samples_by_lineage_aspects(
214
+ self, aspects: List[str], sample_key: str
215
+ ) -> None:
216
+ """Helper method to collect samples for entities that have any of the given lineage aspects.
217
+
218
+ Lineage aspects are stored in JSON format and require quote escaping in LIKE clauses.
219
+ """
220
+ if not aspects:
221
+ return
222
+
223
+ lineage_conditions = []
224
+ for aspect in aspects:
225
+ lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
226
+
227
+ where_clause = " OR ".join(lineage_conditions)
228
+ self._collect_samples_by_subtype(where_clause, sample_key)
229
+
230
+ def _collect_samples_with_all_conditions(self, sample_key: str) -> None:
231
+ """
232
+ Collect samples for entities that have lineage, profiling, and usage aspects.
233
+ These specific 3 cases are added here as these URNs will be shown in the UI. Subject to change in future.
234
+ """
235
+ if not self._lineage_aspects_seen:
236
+ return
237
+ assert self._file_based_dict is not None
238
+
239
+ # Build lineage conditions using the same logic as _collect_samples_by_lineage_aspects
240
+ lineage_conditions = []
241
+ for aspect in self._lineage_aspects_seen:
242
+ lineage_conditions.append(f"aspects LIKE '%\"{aspect}\"%'")
243
+ lineage_where_clause = " OR ".join(lineage_conditions)
244
+
245
+ # Build profiling conditions using the same logic as _collect_samples_by_aspects
246
+ profiling_where_clause = self._build_aspects_where_clause(["datasetProfile"])
247
+
248
+ # Build usage conditions using the same logic as _collect_samples_by_aspects
249
+ usage_where_clause = self._build_aspects_where_clause(
250
+ [
251
+ "datasetUsageStatistics",
252
+ "chartUsageStatistics",
253
+ "dashboardUsageStatistics",
254
+ ]
255
+ )
256
+
257
+ query = f"""
258
+ SELECT urn, subTypes
259
+ FROM urn_aspects
260
+ WHERE ({lineage_where_clause})
261
+ AND ({profiling_where_clause})
262
+ AND ({usage_where_clause})
263
+ limit {self._samples_to_add}
264
+ """
265
+
266
+ for row in self._file_based_dict.sql_query(query):
267
+ sub_type = row["subTypes"] or "unknown"
268
+ self.samples[sample_key][sub_type].append(row["urn"])
269
+
270
+ def _has_fine_grained_lineage(
271
+ self, mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper]
272
+ ) -> bool:
273
+ if isinstance(mcp.aspect, UpstreamLineageClass):
274
+ upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
275
+ if upstream_lineage.fineGrainedLineages:
276
+ return True
277
+ return False
278
+
279
+ def _update_file_based_dict(
280
+ self,
281
+ urn: str,
282
+ entityType: str,
283
+ aspectName: str,
284
+ mcp: Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper],
285
+ ) -> None:
286
+ if is_lineage_aspect(entityType, aspectName):
287
+ self._lineage_aspects_seen.add(aspectName)
288
+ has_fine_grained_lineage = self._has_fine_grained_lineage(mcp)
289
+
290
+ sub_type = "unknown"
291
+ if isinstance(mcp.aspect, SubTypesClass):
292
+ sub_type = mcp.aspect.typeNames[0]
293
+
294
+ assert self._file_based_dict is not None
295
+ if urn in self._file_based_dict:
296
+ if sub_type != "unknown":
297
+ self._file_based_dict[urn].subType = sub_type
298
+ self._file_based_dict[urn].aspects.add(aspectName)
299
+ if has_fine_grained_lineage:
300
+ self._file_based_dict[urn].aspects.add(
301
+ self._fine_grained_lineage_special_case_name
302
+ )
303
+ self._file_based_dict.mark_dirty(urn)
304
+ else:
305
+ self._file_based_dict[urn] = SourceReportSubtypes(
306
+ urn=urn,
307
+ entity_type=entityType,
308
+ subType=sub_type,
309
+ aspects={aspectName}
310
+ if not has_fine_grained_lineage
311
+ else {aspectName, self._fine_grained_lineage_special_case_name},
312
+ )
313
+
160
314
  def _store_workunit_data(self, wu: MetadataWorkUnit) -> None:
161
315
  urn = wu.get_urn()
162
316
 
@@ -169,41 +323,15 @@ class ExamplesReport(Report, Closeable):
169
323
  entityType = mcp.entityType
170
324
  aspectName = mcp.aspectName
171
325
 
172
- if urn not in self._urns_seen:
173
- self._urns_seen.add(urn)
174
- self.entities[entityType].append(urn)
175
-
176
326
  if aspectName is None:
177
327
  continue
178
- self.aspects[entityType][aspectName] += 1
179
- self.aspect_urn_samples[entityType][aspectName].append(urn)
180
- sub_type = "unknown"
181
- if isinstance(mcp.aspect, UpstreamLineageClass):
182
- upstream_lineage = cast(UpstreamLineageClass, mcp.aspect)
183
- if upstream_lineage.fineGrainedLineages:
184
- self.aspect_urn_samples[entityType]["fineGrainedLineages"].append(
185
- urn
186
- )
187
- self.aspects[entityType]["fineGrainedLineages"] += 1
188
- elif isinstance(mcp.aspect, SubTypesClass):
189
- sub_type = mcp.aspect.typeNames[0]
190
- assert self._file_based_dict is not None
191
- if urn in self._file_based_dict:
192
- if sub_type != "unknown":
193
- self._file_based_dict[urn].subType = sub_type
194
- self._file_based_dict[urn].aspects.add(aspectName)
195
- self._file_based_dict.mark_dirty(urn)
196
- else:
197
- self._file_based_dict[urn] = SourceReportSubtypes(
198
- urn=urn,
199
- entity_type=entityType,
200
- subType=sub_type,
201
- aspects={aspectName},
202
- )
328
+
329
+ self._update_file_based_dict(urn, entityType, aspectName, mcp)
203
330
 
204
331
  def compute_stats(self) -> None:
205
332
  if self._file_based_dict is None:
206
333
  return
334
+
207
335
  query = """
208
336
  SELECT entityType, subTypes, aspects, count(*) as count
209
337
  FROM urn_aspects
@@ -223,11 +351,31 @@ class ExamplesReport(Report, Closeable):
223
351
  for aspect in aspects:
224
352
  entity_subtype_aspect_counts[entity_type][sub_type][aspect] += count
225
353
 
354
+ self.aspects.clear()
226
355
  self.aspects_by_subtypes.clear()
356
+ _aspects_seen: Set[str] = set()
227
357
  for entity_type, subtype_counts in entity_subtype_aspect_counts.items():
228
358
  for sub_type, aspect_counts in subtype_counts.items():
359
+ for aspect, count in aspect_counts.items():
360
+ self.aspects[entity_type][aspect] += count
361
+ _aspects_seen.add(aspect)
229
362
  self.aspects_by_subtypes[entity_type][sub_type] = dict(aspect_counts)
230
363
 
364
+ self.samples.clear()
365
+ self._collect_samples_by_aspects(["datasetProfile"], "profiling")
366
+ self._collect_samples_by_aspects(
367
+ [
368
+ "datasetUsageStatistics",
369
+ "chartUsageStatistics",
370
+ "dashboardUsageStatistics",
371
+ ],
372
+ "usage",
373
+ )
374
+ self._collect_samples_by_lineage_aspects(
375
+ list(self._lineage_aspects_seen), "lineage"
376
+ )
377
+ self._collect_samples_with_all_conditions("all_3")
378
+
231
379
 
232
380
  class EntityFilterReport(ReportAttribute):
233
381
  type: str