acryl-datahub 1.2.0.10rc1__py3-none-any.whl → 1.2.0.10rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (29) hide show
  1. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/METADATA +2616 -2616
  2. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/RECORD +29 -29
  3. datahub/_version.py +1 -1
  4. datahub/ingestion/autogenerated/capability_summary.json +12 -0
  5. datahub/ingestion/source/dbt/dbt_common.py +65 -5
  6. datahub/ingestion/source/ge_data_profiler.py +15 -2
  7. datahub/ingestion/source/looker/looker_common.py +75 -74
  8. datahub/ingestion/source/looker/looker_source.py +445 -548
  9. datahub/ingestion/source/looker/lookml_source.py +46 -88
  10. datahub/ingestion/source/redash.py +1 -1
  11. datahub/ingestion/source/superset.py +121 -13
  12. datahub/ingestion/source/tableau/tableau.py +48 -8
  13. datahub/ingestion/source/tableau/tableau_common.py +5 -0
  14. datahub/ingestion/source/tableau/tableau_constant.py +1 -0
  15. datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
  16. datahub/metadata/_internal_schema_classes.py +202 -2
  17. datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
  18. datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +4 -0
  19. datahub/metadata/schema.avsc +98 -2
  20. datahub/metadata/schemas/GlobalSettingsInfo.avsc +72 -0
  21. datahub/metadata/schemas/InstitutionalMemory.avsc +22 -0
  22. datahub/metadata/schemas/LogicalParent.avsc +2 -1
  23. datahub/metadata/schemas/MLModelGroupKey.avsc +2 -1
  24. datahub/metadata/schemas/MetadataChangeEvent.avsc +22 -0
  25. datahub/sdk/dashboard.py +0 -2
  26. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/WHEEL +0 -0
  27. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/entry_points.txt +0 -0
  28. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/licenses/LICENSE +0 -0
  29. {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
- acryl_datahub-1.2.0.10rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
1
+ acryl_datahub-1.2.0.10rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
2
2
  datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
3
3
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
4
- datahub/_version.py,sha256=gbUD5kbOI-y4DH5wBU0PRgKeHjSTCNQ6fgZcMvqwDpg,324
4
+ datahub/_version.py,sha256=VuL0cf5kSPWwJF2YoxiWA-UuOSPyWDPYeQVw6VxtPrk,324
5
5
  datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
6
6
  datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
7
7
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -159,7 +159,7 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
159
159
  datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
160
160
  datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=0BwkpLhORbsiTHq0g_N_1cVVoZYdLR3qz02mNmsV9-M,4444
161
161
  datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
162
- datahub/ingestion/autogenerated/capability_summary.json,sha256=78zS8mOo54IkG9l5KJ_YBXkZmERSSo3pDIrefQx-PWA,110199
162
+ datahub/ingestion/autogenerated/capability_summary.json,sha256=6n3XZj4xPZ3WVfQ29QQvqX5ancSXAqvjaMfpDqFQ8LI,110531
163
163
  datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
164
164
  datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
165
165
  datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -211,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
211
211
  datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
212
212
  datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
213
213
  datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
214
- datahub/ingestion/source/ge_data_profiler.py,sha256=zG3LR_xjc4bz3WxtURWz-lE0rtHE8VjFmOOaPioGTnw,68755
214
+ datahub/ingestion/source/ge_data_profiler.py,sha256=9lEQdLcMBa7znqa6Zz-QWA4Uiv8KiiCALMEERL37pgA,69318
215
215
  datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
216
216
  datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
217
217
  datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
@@ -224,11 +224,11 @@ datahub/ingestion/source/openapi.py,sha256=1eemGG8BM5H8T2OxNSg6kzGDminblLPgPVuPA
224
224
  datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDCyKWnzIrVI5Alo,15420
225
225
  datahub/ingestion/source/preset.py,sha256=fncn-fgYcITsYEHVsvV6cGTQ9_xc_R06ejrw6ZbY3QA,3966
226
226
  datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
227
- datahub/ingestion/source/redash.py,sha256=YxjSad-X_wPmxYH8dJmFz_VCFhiLTCTSlK99WdvcYiA,30653
227
+ datahub/ingestion/source/redash.py,sha256=C4cDikWymbL88fDqaIPX5WA3f2sIEtH7bmhJKkmXJsM,30652
228
228
  datahub/ingestion/source/salesforce.py,sha256=UttN3y4Ylbx_yCFCr-33wUEZFR48nTiYeUfOjGIFj2E,40872
229
229
  datahub/ingestion/source/source_registry.py,sha256=a2mLjJPLkSI-gYCTb_7U7Jo4D8jGknNQ_yScPIihXFk,1208
230
230
  datahub/ingestion/source/sql_queries.py,sha256=Zf6Y84WFCD-j0v4HdtcXshAPrtGq5InncXCjnLaAuS4,14657
231
- datahub/ingestion/source/superset.py,sha256=SLv6DqglPlFJwEskLOqEr-rfgCAb_2AXqWYupk8FqH4,53756
231
+ datahub/ingestion/source/superset.py,sha256=PH6YJZ4rXV2u43-MEapSurgO1joIge6-pklAivY-wzM,57903
232
232
  datahub/ingestion/source/abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
233
233
  datahub/ingestion/source/abs/config.py,sha256=WW9JWbzqAJDblAcJKtNeuBHqOeJsB57lW2PqSD65-BU,6729
234
234
  datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
@@ -302,7 +302,7 @@ datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vB
302
302
  datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
303
303
  datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
304
304
  datahub/ingestion/source/dbt/dbt_cloud.py,sha256=ILkP6ui48VU8pClDQz6jaFR026gcF7lyUX4gt9im8Vg,18428
305
- datahub/ingestion/source/dbt/dbt_common.py,sha256=0JMrN4Gy33_KgG4UY0NQtIS6WIF_MnTm8LZ_fjr5v8I,89475
305
+ datahub/ingestion/source/dbt/dbt_common.py,sha256=OGe9_tDMYuaM_9JZQ3fYtWkao09R5CB5x2SQlD1UDng,91881
306
306
  datahub/ingestion/source/dbt/dbt_core.py,sha256=WVI2ZYXOMxgFzJnJqsqmEGS-5xdfiVIDsCb78lvSeQ0,24930
307
307
  datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpidfGTvOi5fr6I,9839
308
308
  datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -383,7 +383,7 @@ datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfIC
383
383
  datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
384
384
  datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
385
385
  datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
386
- datahub/ingestion/source/looker/looker_common.py,sha256=7o2ZdL1u9d6yHfQt3v0abCouCE_SdDGuBeNDevcAqhY,67884
386
+ datahub/ingestion/source/looker/looker_common.py,sha256=WOOOerQpngT7XkRPD2klOz-2K4_QqCL0hM2Gbs1EUko,67756
387
387
  datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
388
388
  datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
389
389
  datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
@@ -392,14 +392,14 @@ datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01J
392
392
  datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=6smUt_Ya7ZJMHWdGZl3TnhM7XHZVpYQ6gz2i5hHejZ4,11547
393
393
  datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=27WnOuTghayaH-HL4lLoq0IcHvNm1UybMqMnoaxN8Cs,5383
394
394
  datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
395
- datahub/ingestion/source/looker/looker_source.py,sha256=1muVoXk7Jr73D0dlDFSdea8aK2wdyL6wa66qAjyv_OQ,68100
395
+ datahub/ingestion/source/looker/looker_source.py,sha256=a-G_73NWHD0YPDetT-Eyvq5KenJjqbmb-bV5JMOByNU,65048
396
396
  datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
397
397
  datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
398
398
  datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
399
399
  datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
400
400
  datahub/ingestion/source/looker/lookml_config.py,sha256=lulLcjAS1d8ihQseBe4HYn6ALKmJX1vl0H5mxiBzZ74,11395
401
401
  datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
402
- datahub/ingestion/source/looker/lookml_source.py,sha256=iqvem_ND7w-ANcKKSjixM64jRIFOBZX-mNjYMZ_UDxU,44438
402
+ datahub/ingestion/source/looker/lookml_source.py,sha256=aNvEoW0njzqQSoj1KeHiDOBbEGYWWDEcjX-a2RYgqOY,42735
403
403
  datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
404
404
  datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
405
405
  datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
@@ -554,10 +554,10 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
554
554
  datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=DziD57PbHn2Tcy51tYXCG-GQgyTGMUxnkuzVS_xihFY,4079
555
555
  datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
556
556
  datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
557
- datahub/ingestion/source/tableau/tableau.py,sha256=OCdEHTYhJllnvQKalaEdiRATt04syAs2-O1oBYhCu74,156162
558
- datahub/ingestion/source/tableau/tableau_common.py,sha256=4cUm3E8wLfjLSjcAXXWDWVUXAEho_hbsQa2BzAF-vtM,27012
559
- datahub/ingestion/source/tableau/tableau_constant.py,sha256=4ngrOwqxf4cgbLR3i0OKI4pUxmHMABKyywfhXQ0GazA,2592
560
- datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=nSyx9RzC6TCQDm-cTVJ657qT8iDwzk_8JMKpohhmOc4,1046
557
+ datahub/ingestion/source/tableau/tableau.py,sha256=8yJZU3jdAixCloT1EfvzH7hNTv0iEd29gip03SGA2fw,157849
558
+ datahub/ingestion/source/tableau/tableau_common.py,sha256=2vE7DIigPvMNcTCWSou0tliaVy9MgFR1qwqnE4pilw8,27086
559
+ datahub/ingestion/source/tableau/tableau_constant.py,sha256=2WPAHN-GAR83_c3eTTNd8cy0-zC8GIXeUdSxX_mNdas,2608
560
+ datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9_g0aDclKwYkcoxugaWyAcyAMgBCAU,1136
561
561
  datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
562
562
  datahub/ingestion/source/unity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
563
563
  datahub/ingestion/source/unity/analyze_profiler.py,sha256=2pqkFY30CfN4aHgFZZntjeG0hNhBytZJvXC13VfTc1I,4689
@@ -635,8 +635,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
635
635
  datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
636
636
  datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
637
637
  datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
638
- datahub/metadata/_internal_schema_classes.py,sha256=C_RqtHu-dWkZUJWo85PNu9gs4evsVb4vMIbdAyRUM2o,1062642
639
- datahub/metadata/schema.avsc,sha256=Z5bziVmjWxly-mPLjCWic2eXMCkVIYTbWqRFErSK5uk,705606
638
+ datahub/metadata/_internal_schema_classes.py,sha256=ORmPVDR7_UMk8x4pwbTukK7QZm0oFi9XK0DKa0oPjy0,1069945
639
+ datahub/metadata/schema.avsc,sha256=bbtE3veiGe5fqJnMWJTERU99CMOR4CP2lAZidj4UWGE,709640
640
640
  datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
641
641
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
642
642
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -651,7 +651,7 @@ datahub/metadata/com/linkedin/pegasus2avro/application/__init__.py,sha256=1ibbZG
651
651
  datahub/metadata/com/linkedin/pegasus2avro/assertion/__init__.py,sha256=PgK5O-6pVRaEcvmwXAsSkwRLe8NjGiLH8AVBXeArqK8,5751
652
652
  datahub/metadata/com/linkedin/pegasus2avro/businessattribute/__init__.py,sha256=N8kO-eUi0_Rt7weizIExxlnJ2_kZRtPrZLWCC1xtDMA,653
653
653
  datahub/metadata/com/linkedin/pegasus2avro/chart/__init__.py,sha256=RNyyHLBNp_fxgFcBOLWO2UsXR1ofD_JczcBdPEQSusg,848
654
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py,sha256=D5rQ4RNxPzVaZOedwUup9-LMy1pIBhQxL7EQ7jaxvaM,5885
654
+ datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py,sha256=PHRvctPZJ0xxtmr8OMhigEVbky4GQ2Z4JLa63tBNvQg,6037
655
655
  datahub/metadata/com/linkedin/pegasus2avro/common/fieldtransformer/__init__.py,sha256=FN63vLiB3FCmIRqBjTA-0Xt7M6i7h5NhaVzbA1ysv18,396
656
656
  datahub/metadata/com/linkedin/pegasus2avro/connection/__init__.py,sha256=qRtw-dB14pzVzgQ0pDK8kyBplNdpRxVKNj4D70e_FqI,564
657
657
  datahub/metadata/com/linkedin/pegasus2avro/container/__init__.py,sha256=3yWt36KqDKFhRc9pzvt0AMnbMTlhKurGvT3BUvc25QU,510
@@ -709,7 +709,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWezn
709
709
  datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
710
710
  datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
711
711
  datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py,sha256=ASD1mi7q19HVN9QKzbu2T3GxzdDQ_LqZNTehV3SI12c,531
712
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=hKpTDXj2YEBja4hk2fZx3G9yzJ7eCFxcGCbbN0NydTk,933
712
+ datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=kTY87JYIsV2yvBiHE-QrzPc7PIrUFS5aWh_VFQm7tSg,1107
713
713
  datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
714
714
  datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
715
715
  datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
@@ -857,7 +857,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
857
857
  datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
858
858
  datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
859
859
  datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
860
- datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=HKZisxW2HNm6kwJUBaAVol7suJoxSVCjfvWy55HhalQ,11999
860
+ datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=xxwH79mYno0nDt-TZ1bTlEd_8Tv4t01iijYHgPl9838,14927
861
861
  datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
862
862
  datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
863
863
  datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
@@ -874,11 +874,11 @@ datahub/metadata/schemas/IncidentKey.avsc,sha256=Pip__DyNNTal7NxryM3kFi9qHlwntp1
874
874
  datahub/metadata/schemas/IncidentSource.avsc,sha256=lY_SarA3cM55KNENcB5z1Gu2MygxEl9l7R8LdMak9AQ,1199
875
875
  datahub/metadata/schemas/IncidentsSummary.avsc,sha256=NTYp-6Oe92ALApbM3759TJ5pLXRArsSriIPq-f7w9vI,4514
876
876
  datahub/metadata/schemas/InputFields.avsc,sha256=f3SFxnV3-K2D59ctC5H2xCRx8uwqZTRI95KgSotQT-k,33832
877
- datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=0dKK18Gr6eBIgUimZrm9zsBdIfmCgYYfIiWW0w07PoA,4084
877
+ datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=O6I0gs5C1FinXXcYrT7CSEQNoPkBL8A4nFb-1so2TcA,4912
878
878
  datahub/metadata/schemas/IntendedUse.avsc,sha256=IKZSWdvc0uAyyT-FtdQOGbMC-P7RS9cO0vOVKWT6fbw,1361
879
879
  datahub/metadata/schemas/InviteToken.avsc,sha256=8k_9MxHu9GVf7gvS0SlnQu7tJfpbXsRFdz6lQrFKPNc,737
880
880
  datahub/metadata/schemas/InviteTokenKey.avsc,sha256=MuQUlQaeVjaBkjSshB9gsx5Fm0civYgWD8UhCiRLdOQ,434
881
- datahub/metadata/schemas/LogicalParent.avsc,sha256=ocwb2ED-MNNRIPcFFjsFz5Qt7qHeVhFI5ou80zL5XuI,5420
881
+ datahub/metadata/schemas/LogicalParent.avsc,sha256=Wr58mqOVKNc2YIXZSWEjIHqfcJ_U0XH1CRMuMAD5R_Q,5477
882
882
  datahub/metadata/schemas/MLFeatureKey.avsc,sha256=1XFF8P2T3_4mX2oHGY74UJx1LFr6WtTv2jTG9ApjQoA,1150
883
883
  datahub/metadata/schemas/MLFeatureProperties.avsc,sha256=HpF7VcnH2FvDsqy2g5AMJPqU7upkTVADW1ps-9bjINo,6893
884
884
  datahub/metadata/schemas/MLFeatureTableKey.avsc,sha256=hVxNEqsx4GgG11GVryn9ms16OyEzsYdwM5RuAHy_454,1424
@@ -888,14 +888,14 @@ datahub/metadata/schemas/MLMetric.avsc,sha256=y8WPVVwjhu3YGtqpFFJYNYK8w778RRL_d2
888
888
  datahub/metadata/schemas/MLModelDeploymentKey.avsc,sha256=EcA0z4sQfqe3IJ8PO8cGW34XMxc9Q2BbCBjgkJTaznE,2685
889
889
  datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=7IlGrMmX8nfgezvaZyrXskCTCRlwvRzGOYUOpFV3r6Y,5480
890
890
  datahub/metadata/schemas/MLModelFactorPrompts.avsc,sha256=8kX-P4F4mVLFT980z3MwIautt1_6uA-c_Z87nYNDK-k,2712
891
- datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=zIyIHI-23i_oQMbc1sigar9sJNJsa6CYfHHy-nH5IXE,2779
891
+ datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=LFSIgaT-6oTgMO6wzo8bgovCNl6oAIRvjCsRiSDao2I,2808
892
892
  datahub/metadata/schemas/MLModelGroupProperties.avsc,sha256=AZ5Pohk3_pCctQ4hcE1UOURQFYHQne0dw_lRUpOu5WY,6924
893
893
  datahub/metadata/schemas/MLModelKey.avsc,sha256=zwoY9opTL5tMm5aoRHoWcNv5DjERYS-hWR05kVFlcTw,3148
894
894
  datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKjtZsDcTfl2X_jWmtFqo,12355
895
895
  datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=F3lgpMnHBhXsqGncHE9x06P-0RiNCrzbUUWlMkPJxFI,1132
896
896
  datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
897
897
  datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
898
- datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=4FUC9OWiBLHgGmKKfHqOx6XqocwTL0Dj9HQo3-vE84g,380117
898
+ datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=bs_f_quv7oDsJ1vgkkQLaP6rLks2lW2Fq6fE-_fTlh0,381341
899
899
  datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=soCmgrcEBE5yS-mQIm-RIefhb74ONj9Fqayxa0-59KE,13254
900
900
  datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=pT14vUmpj7VJ8hinQ0pcCUtRKx6RAGHWh1eJixkqaE8,12647
901
901
  datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
@@ -961,7 +961,7 @@ datahub/sdk/_shared.py,sha256=uSLPjXfUl_0SPt-kWirkhE6u1CKOC67q5c4hJzxS2uo,28812
961
961
  datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
962
962
  datahub/sdk/chart.py,sha256=_gixCcKp6kCMizWMXwNH1Ip1ZqJ05_Iu2t94dmONQFM,11774
963
963
  datahub/sdk/container.py,sha256=IjnFVGDpSFDvgHuuMb7C3VdBxhJuIMq0q6crOs5PupE,7899
964
- datahub/sdk/dashboard.py,sha256=BtOslP85IfthdnfWzDTSXJKXsTjXP17nIHlYvK7Zqhg,15173
964
+ datahub/sdk/dashboard.py,sha256=ekdchqZ57enVFIfeEOOl0Dk4ec-MFQSncArciTgU1sk,15109
965
965
  datahub/sdk/dataflow.py,sha256=gdAPVVkyKvsKtsa1AwhN_LpzidG_XzV3nhtd1cjnzDA,11128
966
966
  datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
967
967
  datahub/sdk/dataset.py,sha256=-C4TCJAs1PFkLAgkUZEU1JOg3orm7AAIkqjw7oo_4PQ,31400
@@ -1114,8 +1114,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
1114
1114
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
1115
1115
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
1116
1116
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
1117
- acryl_datahub-1.2.0.10rc1.dist-info/METADATA,sha256=RZTwZLgqey2EHL-O4LEZfy5CEJc5WYKTwMzdOl6aoFU,186878
1118
- acryl_datahub-1.2.0.10rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1119
- acryl_datahub-1.2.0.10rc1.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
1120
- acryl_datahub-1.2.0.10rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1121
- acryl_datahub-1.2.0.10rc1.dist-info/RECORD,,
1117
+ acryl_datahub-1.2.0.10rc3.dist-info/METADATA,sha256=dbKMy0Er57vf8j1C1AegAKdo8eZ3dwDDotNwBmJ1akM,186878
1118
+ acryl_datahub-1.2.0.10rc3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
1119
+ acryl_datahub-1.2.0.10rc3.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
1120
+ acryl_datahub-1.2.0.10rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
1121
+ acryl_datahub-1.2.0.10rc3.dist-info/RECORD,,
datahub/_version.py CHANGED
@@ -1,6 +1,6 @@
1
1
  # Published at https://pypi.org/project/acryl-datahub/.
2
2
  __package_name__ = "acryl-datahub"
3
- __version__ = "1.2.0.10rc1"
3
+ __version__ = "1.2.0.10rc3"
4
4
 
5
5
 
6
6
  def is_dev_mode() -> bool:
@@ -2372,6 +2372,12 @@
2372
2372
  "subtype_modifier": null,
2373
2373
  "supported": true
2374
2374
  },
2375
+ {
2376
+ "capability": "TAGS",
2377
+ "description": "Supported by default",
2378
+ "subtype_modifier": null,
2379
+ "supported": true
2380
+ },
2375
2381
  {
2376
2382
  "capability": "LINEAGE_COARSE",
2377
2383
  "description": "Supported by default",
@@ -3174,6 +3180,12 @@
3174
3180
  "subtype_modifier": null,
3175
3181
  "supported": true
3176
3182
  },
3183
+ {
3184
+ "capability": "TAGS",
3185
+ "description": "Supported by default",
3186
+ "subtype_modifier": null,
3187
+ "supported": true
3188
+ },
3177
3189
  {
3178
3190
  "capability": "LINEAGE_COARSE",
3179
3191
  "description": "Supported by default",
@@ -246,6 +246,23 @@ class DBTEntitiesEnabled(ConfigModel):
246
246
  return self.model_performance == EmitDirective.YES
247
247
 
248
248
 
249
+ class MaterializedNodePatternConfig(ConfigModel):
250
+ """Configuration for filtering materialized nodes based on their physical location"""
251
+
252
+ database_pattern: AllowDenyPattern = Field(
253
+ default=AllowDenyPattern.allow_all(),
254
+ description="Regex patterns for database names to filter materialized nodes.",
255
+ )
256
+ schema_pattern: AllowDenyPattern = Field(
257
+ default=AllowDenyPattern.allow_all(),
258
+ description="Regex patterns for schema names in format '{database}.{schema}' to filter materialized nodes.",
259
+ )
260
+ table_pattern: AllowDenyPattern = Field(
261
+ default=AllowDenyPattern.allow_all(),
262
+ description="Regex patterns for table/view names in format '{database}.{schema}.{table}' to filter materialized nodes.",
263
+ )
264
+
265
+
249
266
  class DBTCommonConfig(
250
267
  StatefulIngestionConfigBase,
251
268
  PlatformInstanceConfigMixin,
@@ -294,6 +311,11 @@ class DBTCommonConfig(
294
311
  default=AllowDenyPattern.allow_all(),
295
312
  description="regex patterns for dbt model names to filter in ingestion.",
296
313
  )
314
+ materialized_node_pattern: MaterializedNodePatternConfig = Field(
315
+ default=MaterializedNodePatternConfig(),
316
+ description="Advanced filtering for materialized nodes based on their physical database location. "
317
+ "Provides fine-grained control over database.schema.table patterns for catalog consistency.",
318
+ )
297
319
  meta_mapping: Dict = Field(
298
320
  default={},
299
321
  description="mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.",
@@ -1018,15 +1040,53 @@ class DBTSourceBase(StatefulIngestionSourceBase):
1018
1040
  all_nodes_map,
1019
1041
  )
1020
1042
 
1021
- def _is_allowed_node(self, key: str) -> bool:
1022
- return self.config.node_name_pattern.allowed(key)
1043
+ def _is_allowed_node(self, node: DBTNode) -> bool:
1044
+ """
1045
+ Check whether a node should be processed, using multi-layer rules. Checks for materialized nodes might need to be restricted in the future to some cases
1046
+ """
1047
+ if not self.config.node_name_pattern.allowed(node.dbt_name):
1048
+ return False
1049
+
1050
+ if not self._is_allowed_materialized_node(node):
1051
+ return False
1052
+
1053
+ return True
1054
+
1055
+ def _is_allowed_materialized_node(self, node: DBTNode) -> bool:
1056
+ """Filter nodes based on their materialized database location for catalog consistency"""
1057
+
1058
+ # Database level filtering
1059
+ if not node.database:
1060
+ return True
1061
+ if not self.config.materialized_node_pattern.database_pattern.allowed(
1062
+ node.database
1063
+ ):
1064
+ return False
1065
+
1066
+ # Schema level filtering: {database}.{schema}
1067
+ if not node.schema:
1068
+ return True
1069
+ if not self.config.materialized_node_pattern.schema_pattern.allowed(
1070
+ node._join_parts([node.database, node.schema])
1071
+ ):
1072
+ return False
1073
+
1074
+ # Table level filtering: {database}.{schema}.{table}
1075
+ if not node.name:
1076
+ return True
1077
+ if not self.config.materialized_node_pattern.table_pattern.allowed(
1078
+ node.get_db_fqn()
1079
+ ):
1080
+ return False
1081
+
1082
+ return True
1023
1083
 
1024
1084
  def _filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
1025
1085
  nodes: List[DBTNode] = []
1026
1086
  for node in all_nodes:
1027
1087
  key = node.dbt_name
1028
1088
 
1029
- if not self._is_allowed_node(key):
1089
+ if not self._is_allowed_node(node):
1030
1090
  self.report.nodes_filtered.append(key)
1031
1091
  continue
1032
1092
 
@@ -1118,8 +1178,8 @@ class DBTSourceBase(StatefulIngestionSourceBase):
1118
1178
  cll_nodes.add(dbt_name)
1119
1179
  schema_nodes.add(dbt_name)
1120
1180
 
1121
- for dbt_name in all_nodes_map:
1122
- if self._is_allowed_node(dbt_name):
1181
+ for dbt_name, dbt_node in all_nodes_map.items():
1182
+ if self._is_allowed_node(dbt_node):
1123
1183
  add_node_to_cll_list(dbt_name)
1124
1184
 
1125
1185
  return schema_nodes, cll_nodes
@@ -307,7 +307,6 @@ def _is_single_row_query_method(query: Any) -> bool:
307
307
  "get_column_max",
308
308
  "get_column_mean",
309
309
  "get_column_stdev",
310
- "get_column_nonnull_count",
311
310
  "get_column_unique_count",
312
311
  }
313
312
  CONSTANT_ROW_QUERY_METHODS = {
@@ -331,6 +330,7 @@ def _is_single_row_query_method(query: Any) -> bool:
331
330
 
332
331
  FIRST_PARTY_SINGLE_ROW_QUERY_METHODS = {
333
332
  "get_column_unique_count_dh_patch",
333
+ "_get_column_cardinality",
334
334
  }
335
335
 
336
336
  # We'll do this the inefficient way since the arrays are pretty small.
@@ -497,7 +497,20 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
497
497
  self, column_spec: _SingleColumnSpec, column: str
498
498
  ) -> None:
499
499
  try:
500
- nonnull_count = self.dataset.get_column_nonnull_count(column)
500
+ # Don't use Great Expectations get_column_nonnull_count because it
501
+ # generates this SQL:
502
+ #
503
+ # sum(CASE WHEN (mycolumn IN (NULL) OR mycolumn IS NULL) THEN 1 ELSE 0 END)
504
+ #
505
+ # which fails for complex types (such as Databricks maps) that don't
506
+ # support the IN operator.
507
+ nonnull_count = convert_to_json_serializable(
508
+ self.dataset.engine.execute(
509
+ sa.select(sa.func.count(sa.column(column))).select_from(
510
+ self.dataset._table
511
+ )
512
+ ).scalar()
513
+ )
501
514
  column_spec.nonnull_count = nonnull_count
502
515
  except Exception as e:
503
516
  logger.debug(
@@ -36,7 +36,7 @@ from datahub.api.entities.platformresource.platform_resource import (
36
36
  PlatformResourceKey,
37
37
  )
38
38
  from datahub.emitter.mcp import MetadataChangeProposalWrapper
39
- from datahub.emitter.mcp_builder import ContainerKey, create_embed_mcp
39
+ from datahub.emitter.mcp_builder import ContainerKey
40
40
  from datahub.ingestion.api.report import Report
41
41
  from datahub.ingestion.api.source import SourceReport
42
42
  from datahub.ingestion.source.common.subtypes import DatasetSubTypes
@@ -72,7 +72,6 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
72
72
  UpstreamClass,
73
73
  UpstreamLineage,
74
74
  )
75
- from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
76
75
  from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
77
76
  from datahub.metadata.com.linkedin.pegasus2avro.schema import (
78
77
  ArrayTypeClass,
@@ -90,21 +89,18 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
90
89
  )
91
90
  from datahub.metadata.schema_classes import (
92
91
  BrowsePathEntryClass,
93
- BrowsePathsClass,
94
92
  BrowsePathsV2Class,
95
- ContainerClass,
96
- DatasetPropertiesClass,
93
+ EmbedClass,
97
94
  EnumTypeClass,
98
95
  FineGrainedLineageClass,
99
96
  GlobalTagsClass,
100
97
  SchemaMetadataClass,
101
- StatusClass,
102
- SubTypesClass,
103
98
  TagAssociationClass,
104
99
  TagPropertiesClass,
105
100
  TagSnapshotClass,
106
101
  )
107
102
  from datahub.metadata.urns import TagUrn
103
+ from datahub.sdk.dataset import Dataset
108
104
  from datahub.sql_parsing.sqlglot_lineage import ColumnRef
109
105
  from datahub.utilities.lossy_collections import LossyList, LossySet
110
106
  from datahub.utilities.url_util import remove_port_from_url
@@ -255,6 +251,11 @@ class LookerViewId:
255
251
 
256
252
  return generated_urn
257
253
 
254
+ def get_view_dataset_name(self, config: LookerCommonConfig) -> str:
255
+ n_mapping: ViewNamingPatternMapping = self.get_mapping(config)
256
+ n_mapping.file_path = self.preprocess_file_path(n_mapping.file_path)
257
+ return config.view_naming_pattern.replace_variables(n_mapping)
258
+
258
259
  def get_browse_path(self, config: LookerCommonConfig) -> str:
259
260
  browse_path = config.view_browse_pattern.replace_variables(
260
261
  self.get_mapping(config)
@@ -282,6 +283,22 @@ class LookerViewId:
282
283
  ],
283
284
  )
284
285
 
286
+ def get_view_dataset_parent_container(
287
+ self, config: LookerCommonConfig
288
+ ) -> List[str]:
289
+ project_key = gen_project_key(config, self.project_name)
290
+ view_path = (
291
+ remove_suffix(self.file_path, ".view.lkml")
292
+ if "{file_path}" in config.view_browse_pattern.pattern
293
+ else os.path.dirname(self.file_path)
294
+ )
295
+ path_entries = view_path.split("/") if view_path else []
296
+ return [
297
+ "Develop",
298
+ project_key.as_urn(),
299
+ *path_entries,
300
+ ]
301
+
285
302
 
286
303
  class ViewFieldType(Enum):
287
304
  DIMENSION = "Dimension"
@@ -1286,50 +1303,28 @@ class LookerExplore:
1286
1303
  reporter: SourceReport,
1287
1304
  base_url: str,
1288
1305
  extract_embed_urls: bool,
1289
- ) -> Optional[List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]]:
1290
- # We only generate MCE-s for explores that contain from clauses and do NOT contain joins
1291
- # All other explores (passthrough explores and joins) end in correct resolution of lineage, and don't need additional nodes in the graph.
1292
-
1293
- dataset_snapshot = DatasetSnapshot(
1294
- urn=self.get_explore_urn(config),
1295
- aspects=[], # we append to this list later on
1296
- )
1297
-
1298
- model_key = gen_model_key(config, self.model_name)
1299
- browse_paths = BrowsePathsClass(paths=[self.get_explore_browse_path(config)])
1300
- container = ContainerClass(container=model_key.as_urn())
1301
- dataset_snapshot.aspects.append(browse_paths)
1302
- dataset_snapshot.aspects.append(StatusClass(removed=False))
1303
-
1304
- custom_properties = {
1305
- "project": self.project_name,
1306
- "model": self.model_name,
1307
- "looker.explore.label": self.label,
1308
- "looker.explore.name": self.name,
1309
- "looker.explore.file": self.source_file,
1310
- }
1311
- dataset_props = DatasetPropertiesClass(
1312
- name=str(self.label) if self.label else LookerUtil._display_name(self.name),
1313
- description=self.description,
1314
- customProperties={
1315
- k: str(v) for k, v in custom_properties.items() if v is not None
1316
- },
1317
- )
1318
- dataset_props.externalUrl = self._get_url(base_url)
1306
+ ) -> Dataset:
1307
+ """
1308
+ Generate a Dataset metadata event for this Looker Explore.
1319
1309
 
1320
- dataset_snapshot.aspects.append(dataset_props)
1310
+ Only generates datasets for explores that contain FROM clauses and do NOT contain joins.
1311
+ Passthrough explores and joins are handled via lineage and do not need additional nodes.
1312
+ """
1313
+ upstream_lineage = None
1321
1314
  view_name_to_urn_map: Dict[str, str] = {}
1315
+
1322
1316
  if self.upstream_views is not None:
1323
1317
  assert self.project_name is not None
1324
- upstreams = []
1318
+ upstreams: list[UpstreamClass] = []
1325
1319
  observed_lineage_ts = datetime.datetime.now(tz=datetime.timezone.utc)
1320
+
1326
1321
  for view_ref in sorted(self.upstream_views):
1327
1322
  # set file_path to ViewFieldType.UNKNOWN if file_path is not available to keep backward compatibility
1328
1323
  # if we raise error on file_path equal to None then existing test-cases will fail as mock data
1329
1324
  # doesn't have required attributes.
1330
1325
  file_path: str = (
1331
1326
  cast(str, self.upstream_views_file_path[view_ref.include])
1332
- if self.upstream_views_file_path[view_ref.include] is not None
1327
+ if self.upstream_views_file_path.get(view_ref.include) is not None
1333
1328
  else ViewFieldValue.NOT_AVAILABLE.value
1334
1329
  )
1335
1330
 
@@ -1356,7 +1351,7 @@ class LookerExplore:
1356
1351
  )
1357
1352
  view_name_to_urn_map[view_ref.include] = view_urn
1358
1353
 
1359
- fine_grained_lineages = []
1354
+ fine_grained_lineages: list[FineGrainedLineageClass] = []
1360
1355
  if config.extract_column_level_lineage:
1361
1356
  for field in self.fields or []:
1362
1357
  # Skip creating fine-grained lineage for empty field names to prevent invalid schema field URNs
@@ -1397,9 +1392,11 @@ class LookerExplore:
1397
1392
  )
1398
1393
 
1399
1394
  upstream_lineage = UpstreamLineage(
1400
- upstreams=upstreams, fineGrainedLineages=fine_grained_lineages or None
1395
+ upstreams=upstreams,
1396
+ fineGrainedLineages=fine_grained_lineages or None,
1401
1397
  )
1402
- dataset_snapshot.aspects.append(upstream_lineage)
1398
+
1399
+ schema_metadata = None
1403
1400
  if self.fields is not None:
1404
1401
  schema_metadata = LookerUtil._get_schema(
1405
1402
  platform_name=config.platform_name,
@@ -1407,42 +1404,46 @@ class LookerExplore:
1407
1404
  view_fields=self.fields,
1408
1405
  reporter=reporter,
1409
1406
  )
1410
- if schema_metadata is not None:
1411
- dataset_snapshot.aspects.append(schema_metadata)
1412
-
1413
- mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
1414
- mcp = MetadataChangeProposalWrapper(
1415
- entityUrn=dataset_snapshot.urn,
1416
- aspect=SubTypesClass(typeNames=[DatasetSubTypes.LOOKER_EXPLORE]),
1417
- )
1418
-
1419
- proposals: List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]] = [
1420
- mce,
1421
- mcp,
1422
- ]
1423
1407
 
1424
- # Add tags
1425
- explore_tag_urns: List[TagAssociationClass] = [
1426
- TagAssociationClass(tag=TagUrn(tag).urn()) for tag in self.tags
1427
- ]
1428
- if explore_tag_urns:
1429
- dataset_snapshot.aspects.append(GlobalTagsClass(explore_tag_urns))
1408
+ extra_aspects: List[Union[GlobalTagsClass, EmbedClass]] = []
1430
1409
 
1431
- # If extracting embeds is enabled, produce an MCP for embed URL.
1410
+ explore_tag_urns: List[TagUrn] = [TagUrn(tag) for tag in self.tags]
1432
1411
  if extract_embed_urls:
1433
- embed_mcp = create_embed_mcp(
1434
- dataset_snapshot.urn, self._get_embed_url(base_url)
1435
- )
1436
- proposals.append(embed_mcp)
1412
+ extra_aspects.append(EmbedClass(renderUrl=self._get_embed_url(base_url)))
1437
1413
 
1438
- proposals.append(
1439
- MetadataChangeProposalWrapper(
1440
- entityUrn=dataset_snapshot.urn,
1441
- aspect=container,
1442
- )
1443
- )
1414
+ custom_properties: Dict[str, Optional[str]] = {
1415
+ "project": self.project_name,
1416
+ "model": self.model_name,
1417
+ "looker.explore.label": self.label,
1418
+ "looker.explore.name": self.name,
1419
+ "looker.explore.file": self.source_file,
1420
+ }
1444
1421
 
1445
- return proposals
1422
+ return Dataset(
1423
+ platform=config.platform_name,
1424
+ name=config.explore_naming_pattern.replace_variables(
1425
+ self.get_mapping(config)
1426
+ ),
1427
+ display_name=str(self.label)
1428
+ if self.label
1429
+ else LookerUtil._display_name(self.name),
1430
+ description=self.description,
1431
+ subtype=DatasetSubTypes.LOOKER_EXPLORE,
1432
+ env=config.env,
1433
+ platform_instance=config.platform_instance,
1434
+ custom_properties={
1435
+ k: str(v) for k, v in custom_properties.items() if v is not None
1436
+ },
1437
+ external_url=self._get_url(base_url),
1438
+ upstreams=upstream_lineage,
1439
+ schema=schema_metadata,
1440
+ parent_container=[
1441
+ "Explore",
1442
+ gen_model_key(config, self.model_name).as_urn(),
1443
+ ],
1444
+ tags=explore_tag_urns if explore_tag_urns else None,
1445
+ extra_aspects=extra_aspects,
1446
+ )
1446
1447
 
1447
1448
 
1448
1449
  def gen_project_key(config: LookerCommonConfig, project_name: str) -> LookMLProjectKey: