acryl-datahub 1.2.0.10rc1__py3-none-any.whl → 1.2.0.10rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/METADATA +2616 -2616
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/RECORD +29 -29
- datahub/_version.py +1 -1
- datahub/ingestion/autogenerated/capability_summary.json +12 -0
- datahub/ingestion/source/dbt/dbt_common.py +65 -5
- datahub/ingestion/source/ge_data_profiler.py +15 -2
- datahub/ingestion/source/looker/looker_common.py +75 -74
- datahub/ingestion/source/looker/looker_source.py +445 -548
- datahub/ingestion/source/looker/lookml_source.py +46 -88
- datahub/ingestion/source/redash.py +1 -1
- datahub/ingestion/source/superset.py +121 -13
- datahub/ingestion/source/tableau/tableau.py +48 -8
- datahub/ingestion/source/tableau/tableau_common.py +5 -0
- datahub/ingestion/source/tableau/tableau_constant.py +1 -0
- datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
- datahub/metadata/_internal_schema_classes.py +202 -2
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +4 -0
- datahub/metadata/schema.avsc +98 -2
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +72 -0
- datahub/metadata/schemas/InstitutionalMemory.avsc +22 -0
- datahub/metadata/schemas/LogicalParent.avsc +2 -1
- datahub/metadata/schemas/MLModelGroupKey.avsc +2 -1
- datahub/metadata/schemas/MetadataChangeEvent.avsc +22 -0
- datahub/sdk/dashboard.py +0 -2
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc3.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.2.0.
|
|
1
|
+
acryl_datahub-1.2.0.10rc3.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=VuL0cf5kSPWwJF2YoxiWA-UuOSPyWDPYeQVw6VxtPrk,324
|
|
5
5
|
datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -159,7 +159,7 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
|
159
159
|
datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
|
|
160
160
|
datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=0BwkpLhORbsiTHq0g_N_1cVVoZYdLR3qz02mNmsV9-M,4444
|
|
161
161
|
datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
datahub/ingestion/autogenerated/capability_summary.json,sha256=
|
|
162
|
+
datahub/ingestion/autogenerated/capability_summary.json,sha256=6n3XZj4xPZ3WVfQ29QQvqX5ancSXAqvjaMfpDqFQ8LI,110531
|
|
163
163
|
datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
|
|
164
164
|
datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
|
|
165
165
|
datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -211,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
|
|
|
211
211
|
datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
|
|
212
212
|
datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
|
|
213
213
|
datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
|
|
214
|
-
datahub/ingestion/source/ge_data_profiler.py,sha256=
|
|
214
|
+
datahub/ingestion/source/ge_data_profiler.py,sha256=9lEQdLcMBa7znqa6Zz-QWA4Uiv8KiiCALMEERL37pgA,69318
|
|
215
215
|
datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
|
|
216
216
|
datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
|
|
217
217
|
datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
|
|
@@ -224,11 +224,11 @@ datahub/ingestion/source/openapi.py,sha256=1eemGG8BM5H8T2OxNSg6kzGDminblLPgPVuPA
|
|
|
224
224
|
datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDCyKWnzIrVI5Alo,15420
|
|
225
225
|
datahub/ingestion/source/preset.py,sha256=fncn-fgYcITsYEHVsvV6cGTQ9_xc_R06ejrw6ZbY3QA,3966
|
|
226
226
|
datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
|
|
227
|
-
datahub/ingestion/source/redash.py,sha256=
|
|
227
|
+
datahub/ingestion/source/redash.py,sha256=C4cDikWymbL88fDqaIPX5WA3f2sIEtH7bmhJKkmXJsM,30652
|
|
228
228
|
datahub/ingestion/source/salesforce.py,sha256=UttN3y4Ylbx_yCFCr-33wUEZFR48nTiYeUfOjGIFj2E,40872
|
|
229
229
|
datahub/ingestion/source/source_registry.py,sha256=a2mLjJPLkSI-gYCTb_7U7Jo4D8jGknNQ_yScPIihXFk,1208
|
|
230
230
|
datahub/ingestion/source/sql_queries.py,sha256=Zf6Y84WFCD-j0v4HdtcXshAPrtGq5InncXCjnLaAuS4,14657
|
|
231
|
-
datahub/ingestion/source/superset.py,sha256=
|
|
231
|
+
datahub/ingestion/source/superset.py,sha256=PH6YJZ4rXV2u43-MEapSurgO1joIge6-pklAivY-wzM,57903
|
|
232
232
|
datahub/ingestion/source/abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
233
233
|
datahub/ingestion/source/abs/config.py,sha256=WW9JWbzqAJDblAcJKtNeuBHqOeJsB57lW2PqSD65-BU,6729
|
|
234
234
|
datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
|
|
@@ -302,7 +302,7 @@ datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vB
|
|
|
302
302
|
datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
|
|
303
303
|
datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
304
304
|
datahub/ingestion/source/dbt/dbt_cloud.py,sha256=ILkP6ui48VU8pClDQz6jaFR026gcF7lyUX4gt9im8Vg,18428
|
|
305
|
-
datahub/ingestion/source/dbt/dbt_common.py,sha256=
|
|
305
|
+
datahub/ingestion/source/dbt/dbt_common.py,sha256=OGe9_tDMYuaM_9JZQ3fYtWkao09R5CB5x2SQlD1UDng,91881
|
|
306
306
|
datahub/ingestion/source/dbt/dbt_core.py,sha256=WVI2ZYXOMxgFzJnJqsqmEGS-5xdfiVIDsCb78lvSeQ0,24930
|
|
307
307
|
datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpidfGTvOi5fr6I,9839
|
|
308
308
|
datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -383,7 +383,7 @@ datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfIC
|
|
|
383
383
|
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
|
|
384
384
|
datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
385
385
|
datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
|
|
386
|
-
datahub/ingestion/source/looker/looker_common.py,sha256=
|
|
386
|
+
datahub/ingestion/source/looker/looker_common.py,sha256=WOOOerQpngT7XkRPD2klOz-2K4_QqCL0hM2Gbs1EUko,67756
|
|
387
387
|
datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
|
|
388
388
|
datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
|
|
389
389
|
datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
|
|
@@ -392,14 +392,14 @@ datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01J
|
|
|
392
392
|
datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=6smUt_Ya7ZJMHWdGZl3TnhM7XHZVpYQ6gz2i5hHejZ4,11547
|
|
393
393
|
datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=27WnOuTghayaH-HL4lLoq0IcHvNm1UybMqMnoaxN8Cs,5383
|
|
394
394
|
datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
|
|
395
|
-
datahub/ingestion/source/looker/looker_source.py,sha256=
|
|
395
|
+
datahub/ingestion/source/looker/looker_source.py,sha256=a-G_73NWHD0YPDetT-Eyvq5KenJjqbmb-bV5JMOByNU,65048
|
|
396
396
|
datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
|
|
397
397
|
datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
|
|
398
398
|
datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
|
|
399
399
|
datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
|
|
400
400
|
datahub/ingestion/source/looker/lookml_config.py,sha256=lulLcjAS1d8ihQseBe4HYn6ALKmJX1vl0H5mxiBzZ74,11395
|
|
401
401
|
datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
|
|
402
|
-
datahub/ingestion/source/looker/lookml_source.py,sha256=
|
|
402
|
+
datahub/ingestion/source/looker/lookml_source.py,sha256=aNvEoW0njzqQSoj1KeHiDOBbEGYWWDEcjX-a2RYgqOY,42735
|
|
403
403
|
datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
|
|
404
404
|
datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
|
|
405
405
|
datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
|
|
@@ -554,10 +554,10 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
|
|
|
554
554
|
datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=DziD57PbHn2Tcy51tYXCG-GQgyTGMUxnkuzVS_xihFY,4079
|
|
555
555
|
datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
|
|
556
556
|
datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
557
|
-
datahub/ingestion/source/tableau/tableau.py,sha256=
|
|
558
|
-
datahub/ingestion/source/tableau/tableau_common.py,sha256=
|
|
559
|
-
datahub/ingestion/source/tableau/tableau_constant.py,sha256=
|
|
560
|
-
datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=
|
|
557
|
+
datahub/ingestion/source/tableau/tableau.py,sha256=8yJZU3jdAixCloT1EfvzH7hNTv0iEd29gip03SGA2fw,157849
|
|
558
|
+
datahub/ingestion/source/tableau/tableau_common.py,sha256=2vE7DIigPvMNcTCWSou0tliaVy9MgFR1qwqnE4pilw8,27086
|
|
559
|
+
datahub/ingestion/source/tableau/tableau_constant.py,sha256=2WPAHN-GAR83_c3eTTNd8cy0-zC8GIXeUdSxX_mNdas,2608
|
|
560
|
+
datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9_g0aDclKwYkcoxugaWyAcyAMgBCAU,1136
|
|
561
561
|
datahub/ingestion/source/tableau/tableau_validation.py,sha256=Hjbfc1AMIkGgzo5ffWXtNRjrxSxzHvw7-dYZDt4d3WE,1819
|
|
562
562
|
datahub/ingestion/source/unity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
563
563
|
datahub/ingestion/source/unity/analyze_profiler.py,sha256=2pqkFY30CfN4aHgFZZntjeG0hNhBytZJvXC13VfTc1I,4689
|
|
@@ -635,8 +635,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
|
|
|
635
635
|
datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
|
|
636
636
|
datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
|
|
637
637
|
datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
|
|
638
|
-
datahub/metadata/_internal_schema_classes.py,sha256=
|
|
639
|
-
datahub/metadata/schema.avsc,sha256=
|
|
638
|
+
datahub/metadata/_internal_schema_classes.py,sha256=ORmPVDR7_UMk8x4pwbTukK7QZm0oFi9XK0DKa0oPjy0,1069945
|
|
639
|
+
datahub/metadata/schema.avsc,sha256=bbtE3veiGe5fqJnMWJTERU99CMOR4CP2lAZidj4UWGE,709640
|
|
640
640
|
datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
|
|
641
641
|
datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
|
|
642
642
|
datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
|
|
@@ -651,7 +651,7 @@ datahub/metadata/com/linkedin/pegasus2avro/application/__init__.py,sha256=1ibbZG
|
|
|
651
651
|
datahub/metadata/com/linkedin/pegasus2avro/assertion/__init__.py,sha256=PgK5O-6pVRaEcvmwXAsSkwRLe8NjGiLH8AVBXeArqK8,5751
|
|
652
652
|
datahub/metadata/com/linkedin/pegasus2avro/businessattribute/__init__.py,sha256=N8kO-eUi0_Rt7weizIExxlnJ2_kZRtPrZLWCC1xtDMA,653
|
|
653
653
|
datahub/metadata/com/linkedin/pegasus2avro/chart/__init__.py,sha256=RNyyHLBNp_fxgFcBOLWO2UsXR1ofD_JczcBdPEQSusg,848
|
|
654
|
-
datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py,sha256=
|
|
654
|
+
datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py,sha256=PHRvctPZJ0xxtmr8OMhigEVbky4GQ2Z4JLa63tBNvQg,6037
|
|
655
655
|
datahub/metadata/com/linkedin/pegasus2avro/common/fieldtransformer/__init__.py,sha256=FN63vLiB3FCmIRqBjTA-0Xt7M6i7h5NhaVzbA1ysv18,396
|
|
656
656
|
datahub/metadata/com/linkedin/pegasus2avro/connection/__init__.py,sha256=qRtw-dB14pzVzgQ0pDK8kyBplNdpRxVKNj4D70e_FqI,564
|
|
657
657
|
datahub/metadata/com/linkedin/pegasus2avro/container/__init__.py,sha256=3yWt36KqDKFhRc9pzvt0AMnbMTlhKurGvT3BUvc25QU,510
|
|
@@ -709,7 +709,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWezn
|
|
|
709
709
|
datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
|
|
710
710
|
datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
|
|
711
711
|
datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py,sha256=ASD1mi7q19HVN9QKzbu2T3GxzdDQ_LqZNTehV3SI12c,531
|
|
712
|
-
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=
|
|
712
|
+
datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=kTY87JYIsV2yvBiHE-QrzPc7PIrUFS5aWh_VFQm7tSg,1107
|
|
713
713
|
datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
|
|
714
714
|
datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
|
|
715
715
|
datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
|
|
@@ -857,7 +857,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
|
|
|
857
857
|
datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
|
|
858
858
|
datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
|
|
859
859
|
datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
|
|
860
|
-
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=
|
|
860
|
+
datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=xxwH79mYno0nDt-TZ1bTlEd_8Tv4t01iijYHgPl9838,14927
|
|
861
861
|
datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
|
|
862
862
|
datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
|
|
863
863
|
datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
|
|
@@ -874,11 +874,11 @@ datahub/metadata/schemas/IncidentKey.avsc,sha256=Pip__DyNNTal7NxryM3kFi9qHlwntp1
|
|
|
874
874
|
datahub/metadata/schemas/IncidentSource.avsc,sha256=lY_SarA3cM55KNENcB5z1Gu2MygxEl9l7R8LdMak9AQ,1199
|
|
875
875
|
datahub/metadata/schemas/IncidentsSummary.avsc,sha256=NTYp-6Oe92ALApbM3759TJ5pLXRArsSriIPq-f7w9vI,4514
|
|
876
876
|
datahub/metadata/schemas/InputFields.avsc,sha256=f3SFxnV3-K2D59ctC5H2xCRx8uwqZTRI95KgSotQT-k,33832
|
|
877
|
-
datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=
|
|
877
|
+
datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=O6I0gs5C1FinXXcYrT7CSEQNoPkBL8A4nFb-1so2TcA,4912
|
|
878
878
|
datahub/metadata/schemas/IntendedUse.avsc,sha256=IKZSWdvc0uAyyT-FtdQOGbMC-P7RS9cO0vOVKWT6fbw,1361
|
|
879
879
|
datahub/metadata/schemas/InviteToken.avsc,sha256=8k_9MxHu9GVf7gvS0SlnQu7tJfpbXsRFdz6lQrFKPNc,737
|
|
880
880
|
datahub/metadata/schemas/InviteTokenKey.avsc,sha256=MuQUlQaeVjaBkjSshB9gsx5Fm0civYgWD8UhCiRLdOQ,434
|
|
881
|
-
datahub/metadata/schemas/LogicalParent.avsc,sha256=
|
|
881
|
+
datahub/metadata/schemas/LogicalParent.avsc,sha256=Wr58mqOVKNc2YIXZSWEjIHqfcJ_U0XH1CRMuMAD5R_Q,5477
|
|
882
882
|
datahub/metadata/schemas/MLFeatureKey.avsc,sha256=1XFF8P2T3_4mX2oHGY74UJx1LFr6WtTv2jTG9ApjQoA,1150
|
|
883
883
|
datahub/metadata/schemas/MLFeatureProperties.avsc,sha256=HpF7VcnH2FvDsqy2g5AMJPqU7upkTVADW1ps-9bjINo,6893
|
|
884
884
|
datahub/metadata/schemas/MLFeatureTableKey.avsc,sha256=hVxNEqsx4GgG11GVryn9ms16OyEzsYdwM5RuAHy_454,1424
|
|
@@ -888,14 +888,14 @@ datahub/metadata/schemas/MLMetric.avsc,sha256=y8WPVVwjhu3YGtqpFFJYNYK8w778RRL_d2
|
|
|
888
888
|
datahub/metadata/schemas/MLModelDeploymentKey.avsc,sha256=EcA0z4sQfqe3IJ8PO8cGW34XMxc9Q2BbCBjgkJTaznE,2685
|
|
889
889
|
datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=7IlGrMmX8nfgezvaZyrXskCTCRlwvRzGOYUOpFV3r6Y,5480
|
|
890
890
|
datahub/metadata/schemas/MLModelFactorPrompts.avsc,sha256=8kX-P4F4mVLFT980z3MwIautt1_6uA-c_Z87nYNDK-k,2712
|
|
891
|
-
datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=
|
|
891
|
+
datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=LFSIgaT-6oTgMO6wzo8bgovCNl6oAIRvjCsRiSDao2I,2808
|
|
892
892
|
datahub/metadata/schemas/MLModelGroupProperties.avsc,sha256=AZ5Pohk3_pCctQ4hcE1UOURQFYHQne0dw_lRUpOu5WY,6924
|
|
893
893
|
datahub/metadata/schemas/MLModelKey.avsc,sha256=zwoY9opTL5tMm5aoRHoWcNv5DjERYS-hWR05kVFlcTw,3148
|
|
894
894
|
datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKjtZsDcTfl2X_jWmtFqo,12355
|
|
895
895
|
datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=F3lgpMnHBhXsqGncHE9x06P-0RiNCrzbUUWlMkPJxFI,1132
|
|
896
896
|
datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
|
|
897
897
|
datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
|
|
898
|
-
datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=
|
|
898
|
+
datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=bs_f_quv7oDsJ1vgkkQLaP6rLks2lW2Fq6fE-_fTlh0,381341
|
|
899
899
|
datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=soCmgrcEBE5yS-mQIm-RIefhb74ONj9Fqayxa0-59KE,13254
|
|
900
900
|
datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=pT14vUmpj7VJ8hinQ0pcCUtRKx6RAGHWh1eJixkqaE8,12647
|
|
901
901
|
datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
|
|
@@ -961,7 +961,7 @@ datahub/sdk/_shared.py,sha256=uSLPjXfUl_0SPt-kWirkhE6u1CKOC67q5c4hJzxS2uo,28812
|
|
|
961
961
|
datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
|
|
962
962
|
datahub/sdk/chart.py,sha256=_gixCcKp6kCMizWMXwNH1Ip1ZqJ05_Iu2t94dmONQFM,11774
|
|
963
963
|
datahub/sdk/container.py,sha256=IjnFVGDpSFDvgHuuMb7C3VdBxhJuIMq0q6crOs5PupE,7899
|
|
964
|
-
datahub/sdk/dashboard.py,sha256=
|
|
964
|
+
datahub/sdk/dashboard.py,sha256=ekdchqZ57enVFIfeEOOl0Dk4ec-MFQSncArciTgU1sk,15109
|
|
965
965
|
datahub/sdk/dataflow.py,sha256=gdAPVVkyKvsKtsa1AwhN_LpzidG_XzV3nhtd1cjnzDA,11128
|
|
966
966
|
datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
|
|
967
967
|
datahub/sdk/dataset.py,sha256=-C4TCJAs1PFkLAgkUZEU1JOg3orm7AAIkqjw7oo_4PQ,31400
|
|
@@ -1114,8 +1114,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1114
1114
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1115
1115
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1116
1116
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1117
|
-
acryl_datahub-1.2.0.
|
|
1118
|
-
acryl_datahub-1.2.0.
|
|
1119
|
-
acryl_datahub-1.2.0.
|
|
1120
|
-
acryl_datahub-1.2.0.
|
|
1121
|
-
acryl_datahub-1.2.0.
|
|
1117
|
+
acryl_datahub-1.2.0.10rc3.dist-info/METADATA,sha256=dbKMy0Er57vf8j1C1AegAKdo8eZ3dwDDotNwBmJ1akM,186878
|
|
1118
|
+
acryl_datahub-1.2.0.10rc3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1119
|
+
acryl_datahub-1.2.0.10rc3.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
|
|
1120
|
+
acryl_datahub-1.2.0.10rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1121
|
+
acryl_datahub-1.2.0.10rc3.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
|
@@ -2372,6 +2372,12 @@
|
|
|
2372
2372
|
"subtype_modifier": null,
|
|
2373
2373
|
"supported": true
|
|
2374
2374
|
},
|
|
2375
|
+
{
|
|
2376
|
+
"capability": "TAGS",
|
|
2377
|
+
"description": "Supported by default",
|
|
2378
|
+
"subtype_modifier": null,
|
|
2379
|
+
"supported": true
|
|
2380
|
+
},
|
|
2375
2381
|
{
|
|
2376
2382
|
"capability": "LINEAGE_COARSE",
|
|
2377
2383
|
"description": "Supported by default",
|
|
@@ -3174,6 +3180,12 @@
|
|
|
3174
3180
|
"subtype_modifier": null,
|
|
3175
3181
|
"supported": true
|
|
3176
3182
|
},
|
|
3183
|
+
{
|
|
3184
|
+
"capability": "TAGS",
|
|
3185
|
+
"description": "Supported by default",
|
|
3186
|
+
"subtype_modifier": null,
|
|
3187
|
+
"supported": true
|
|
3188
|
+
},
|
|
3177
3189
|
{
|
|
3178
3190
|
"capability": "LINEAGE_COARSE",
|
|
3179
3191
|
"description": "Supported by default",
|
|
@@ -246,6 +246,23 @@ class DBTEntitiesEnabled(ConfigModel):
|
|
|
246
246
|
return self.model_performance == EmitDirective.YES
|
|
247
247
|
|
|
248
248
|
|
|
249
|
+
class MaterializedNodePatternConfig(ConfigModel):
|
|
250
|
+
"""Configuration for filtering materialized nodes based on their physical location"""
|
|
251
|
+
|
|
252
|
+
database_pattern: AllowDenyPattern = Field(
|
|
253
|
+
default=AllowDenyPattern.allow_all(),
|
|
254
|
+
description="Regex patterns for database names to filter materialized nodes.",
|
|
255
|
+
)
|
|
256
|
+
schema_pattern: AllowDenyPattern = Field(
|
|
257
|
+
default=AllowDenyPattern.allow_all(),
|
|
258
|
+
description="Regex patterns for schema names in format '{database}.{schema}' to filter materialized nodes.",
|
|
259
|
+
)
|
|
260
|
+
table_pattern: AllowDenyPattern = Field(
|
|
261
|
+
default=AllowDenyPattern.allow_all(),
|
|
262
|
+
description="Regex patterns for table/view names in format '{database}.{schema}.{table}' to filter materialized nodes.",
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
249
266
|
class DBTCommonConfig(
|
|
250
267
|
StatefulIngestionConfigBase,
|
|
251
268
|
PlatformInstanceConfigMixin,
|
|
@@ -294,6 +311,11 @@ class DBTCommonConfig(
|
|
|
294
311
|
default=AllowDenyPattern.allow_all(),
|
|
295
312
|
description="regex patterns for dbt model names to filter in ingestion.",
|
|
296
313
|
)
|
|
314
|
+
materialized_node_pattern: MaterializedNodePatternConfig = Field(
|
|
315
|
+
default=MaterializedNodePatternConfig(),
|
|
316
|
+
description="Advanced filtering for materialized nodes based on their physical database location. "
|
|
317
|
+
"Provides fine-grained control over database.schema.table patterns for catalog consistency.",
|
|
318
|
+
)
|
|
297
319
|
meta_mapping: Dict = Field(
|
|
298
320
|
default={},
|
|
299
321
|
description="mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.",
|
|
@@ -1018,15 +1040,53 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
|
|
1018
1040
|
all_nodes_map,
|
|
1019
1041
|
)
|
|
1020
1042
|
|
|
1021
|
-
def _is_allowed_node(self,
|
|
1022
|
-
|
|
1043
|
+
def _is_allowed_node(self, node: DBTNode) -> bool:
|
|
1044
|
+
"""
|
|
1045
|
+
Check whether a node should be processed, using multi-layer rules. Checks for materialized nodes might need to be restricted in the future to some cases
|
|
1046
|
+
"""
|
|
1047
|
+
if not self.config.node_name_pattern.allowed(node.dbt_name):
|
|
1048
|
+
return False
|
|
1049
|
+
|
|
1050
|
+
if not self._is_allowed_materialized_node(node):
|
|
1051
|
+
return False
|
|
1052
|
+
|
|
1053
|
+
return True
|
|
1054
|
+
|
|
1055
|
+
def _is_allowed_materialized_node(self, node: DBTNode) -> bool:
|
|
1056
|
+
"""Filter nodes based on their materialized database location for catalog consistency"""
|
|
1057
|
+
|
|
1058
|
+
# Database level filtering
|
|
1059
|
+
if not node.database:
|
|
1060
|
+
return True
|
|
1061
|
+
if not self.config.materialized_node_pattern.database_pattern.allowed(
|
|
1062
|
+
node.database
|
|
1063
|
+
):
|
|
1064
|
+
return False
|
|
1065
|
+
|
|
1066
|
+
# Schema level filtering: {database}.{schema}
|
|
1067
|
+
if not node.schema:
|
|
1068
|
+
return True
|
|
1069
|
+
if not self.config.materialized_node_pattern.schema_pattern.allowed(
|
|
1070
|
+
node._join_parts([node.database, node.schema])
|
|
1071
|
+
):
|
|
1072
|
+
return False
|
|
1073
|
+
|
|
1074
|
+
# Table level filtering: {database}.{schema}.{table}
|
|
1075
|
+
if not node.name:
|
|
1076
|
+
return True
|
|
1077
|
+
if not self.config.materialized_node_pattern.table_pattern.allowed(
|
|
1078
|
+
node.get_db_fqn()
|
|
1079
|
+
):
|
|
1080
|
+
return False
|
|
1081
|
+
|
|
1082
|
+
return True
|
|
1023
1083
|
|
|
1024
1084
|
def _filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
|
|
1025
1085
|
nodes: List[DBTNode] = []
|
|
1026
1086
|
for node in all_nodes:
|
|
1027
1087
|
key = node.dbt_name
|
|
1028
1088
|
|
|
1029
|
-
if not self._is_allowed_node(
|
|
1089
|
+
if not self._is_allowed_node(node):
|
|
1030
1090
|
self.report.nodes_filtered.append(key)
|
|
1031
1091
|
continue
|
|
1032
1092
|
|
|
@@ -1118,8 +1178,8 @@ class DBTSourceBase(StatefulIngestionSourceBase):
|
|
|
1118
1178
|
cll_nodes.add(dbt_name)
|
|
1119
1179
|
schema_nodes.add(dbt_name)
|
|
1120
1180
|
|
|
1121
|
-
for dbt_name in all_nodes_map:
|
|
1122
|
-
if self._is_allowed_node(
|
|
1181
|
+
for dbt_name, dbt_node in all_nodes_map.items():
|
|
1182
|
+
if self._is_allowed_node(dbt_node):
|
|
1123
1183
|
add_node_to_cll_list(dbt_name)
|
|
1124
1184
|
|
|
1125
1185
|
return schema_nodes, cll_nodes
|
|
@@ -307,7 +307,6 @@ def _is_single_row_query_method(query: Any) -> bool:
|
|
|
307
307
|
"get_column_max",
|
|
308
308
|
"get_column_mean",
|
|
309
309
|
"get_column_stdev",
|
|
310
|
-
"get_column_nonnull_count",
|
|
311
310
|
"get_column_unique_count",
|
|
312
311
|
}
|
|
313
312
|
CONSTANT_ROW_QUERY_METHODS = {
|
|
@@ -331,6 +330,7 @@ def _is_single_row_query_method(query: Any) -> bool:
|
|
|
331
330
|
|
|
332
331
|
FIRST_PARTY_SINGLE_ROW_QUERY_METHODS = {
|
|
333
332
|
"get_column_unique_count_dh_patch",
|
|
333
|
+
"_get_column_cardinality",
|
|
334
334
|
}
|
|
335
335
|
|
|
336
336
|
# We'll do this the inefficient way since the arrays are pretty small.
|
|
@@ -497,7 +497,20 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
|
|
|
497
497
|
self, column_spec: _SingleColumnSpec, column: str
|
|
498
498
|
) -> None:
|
|
499
499
|
try:
|
|
500
|
-
|
|
500
|
+
# Don't use Great Expectations get_column_nonnull_count because it
|
|
501
|
+
# generates this SQL:
|
|
502
|
+
#
|
|
503
|
+
# sum(CASE WHEN (mycolumn IN (NULL) OR mycolumn IS NULL) THEN 1 ELSE 0 END)
|
|
504
|
+
#
|
|
505
|
+
# which fails for complex types (such as Databricks maps) that don't
|
|
506
|
+
# support the IN operator.
|
|
507
|
+
nonnull_count = convert_to_json_serializable(
|
|
508
|
+
self.dataset.engine.execute(
|
|
509
|
+
sa.select(sa.func.count(sa.column(column))).select_from(
|
|
510
|
+
self.dataset._table
|
|
511
|
+
)
|
|
512
|
+
).scalar()
|
|
513
|
+
)
|
|
501
514
|
column_spec.nonnull_count = nonnull_count
|
|
502
515
|
except Exception as e:
|
|
503
516
|
logger.debug(
|
|
@@ -36,7 +36,7 @@ from datahub.api.entities.platformresource.platform_resource import (
|
|
|
36
36
|
PlatformResourceKey,
|
|
37
37
|
)
|
|
38
38
|
from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
|
39
|
-
from datahub.emitter.mcp_builder import ContainerKey
|
|
39
|
+
from datahub.emitter.mcp_builder import ContainerKey
|
|
40
40
|
from datahub.ingestion.api.report import Report
|
|
41
41
|
from datahub.ingestion.api.source import SourceReport
|
|
42
42
|
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
|
|
@@ -72,7 +72,6 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
|
|
|
72
72
|
UpstreamClass,
|
|
73
73
|
UpstreamLineage,
|
|
74
74
|
)
|
|
75
|
-
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
|
|
76
75
|
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
|
|
77
76
|
from datahub.metadata.com.linkedin.pegasus2avro.schema import (
|
|
78
77
|
ArrayTypeClass,
|
|
@@ -90,21 +89,18 @@ from datahub.metadata.com.linkedin.pegasus2avro.schema import (
|
|
|
90
89
|
)
|
|
91
90
|
from datahub.metadata.schema_classes import (
|
|
92
91
|
BrowsePathEntryClass,
|
|
93
|
-
BrowsePathsClass,
|
|
94
92
|
BrowsePathsV2Class,
|
|
95
|
-
|
|
96
|
-
DatasetPropertiesClass,
|
|
93
|
+
EmbedClass,
|
|
97
94
|
EnumTypeClass,
|
|
98
95
|
FineGrainedLineageClass,
|
|
99
96
|
GlobalTagsClass,
|
|
100
97
|
SchemaMetadataClass,
|
|
101
|
-
StatusClass,
|
|
102
|
-
SubTypesClass,
|
|
103
98
|
TagAssociationClass,
|
|
104
99
|
TagPropertiesClass,
|
|
105
100
|
TagSnapshotClass,
|
|
106
101
|
)
|
|
107
102
|
from datahub.metadata.urns import TagUrn
|
|
103
|
+
from datahub.sdk.dataset import Dataset
|
|
108
104
|
from datahub.sql_parsing.sqlglot_lineage import ColumnRef
|
|
109
105
|
from datahub.utilities.lossy_collections import LossyList, LossySet
|
|
110
106
|
from datahub.utilities.url_util import remove_port_from_url
|
|
@@ -255,6 +251,11 @@ class LookerViewId:
|
|
|
255
251
|
|
|
256
252
|
return generated_urn
|
|
257
253
|
|
|
254
|
+
def get_view_dataset_name(self, config: LookerCommonConfig) -> str:
|
|
255
|
+
n_mapping: ViewNamingPatternMapping = self.get_mapping(config)
|
|
256
|
+
n_mapping.file_path = self.preprocess_file_path(n_mapping.file_path)
|
|
257
|
+
return config.view_naming_pattern.replace_variables(n_mapping)
|
|
258
|
+
|
|
258
259
|
def get_browse_path(self, config: LookerCommonConfig) -> str:
|
|
259
260
|
browse_path = config.view_browse_pattern.replace_variables(
|
|
260
261
|
self.get_mapping(config)
|
|
@@ -282,6 +283,22 @@ class LookerViewId:
|
|
|
282
283
|
],
|
|
283
284
|
)
|
|
284
285
|
|
|
286
|
+
def get_view_dataset_parent_container(
|
|
287
|
+
self, config: LookerCommonConfig
|
|
288
|
+
) -> List[str]:
|
|
289
|
+
project_key = gen_project_key(config, self.project_name)
|
|
290
|
+
view_path = (
|
|
291
|
+
remove_suffix(self.file_path, ".view.lkml")
|
|
292
|
+
if "{file_path}" in config.view_browse_pattern.pattern
|
|
293
|
+
else os.path.dirname(self.file_path)
|
|
294
|
+
)
|
|
295
|
+
path_entries = view_path.split("/") if view_path else []
|
|
296
|
+
return [
|
|
297
|
+
"Develop",
|
|
298
|
+
project_key.as_urn(),
|
|
299
|
+
*path_entries,
|
|
300
|
+
]
|
|
301
|
+
|
|
285
302
|
|
|
286
303
|
class ViewFieldType(Enum):
|
|
287
304
|
DIMENSION = "Dimension"
|
|
@@ -1286,50 +1303,28 @@ class LookerExplore:
|
|
|
1286
1303
|
reporter: SourceReport,
|
|
1287
1304
|
base_url: str,
|
|
1288
1305
|
extract_embed_urls: bool,
|
|
1289
|
-
) ->
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
dataset_snapshot = DatasetSnapshot(
|
|
1294
|
-
urn=self.get_explore_urn(config),
|
|
1295
|
-
aspects=[], # we append to this list later on
|
|
1296
|
-
)
|
|
1297
|
-
|
|
1298
|
-
model_key = gen_model_key(config, self.model_name)
|
|
1299
|
-
browse_paths = BrowsePathsClass(paths=[self.get_explore_browse_path(config)])
|
|
1300
|
-
container = ContainerClass(container=model_key.as_urn())
|
|
1301
|
-
dataset_snapshot.aspects.append(browse_paths)
|
|
1302
|
-
dataset_snapshot.aspects.append(StatusClass(removed=False))
|
|
1303
|
-
|
|
1304
|
-
custom_properties = {
|
|
1305
|
-
"project": self.project_name,
|
|
1306
|
-
"model": self.model_name,
|
|
1307
|
-
"looker.explore.label": self.label,
|
|
1308
|
-
"looker.explore.name": self.name,
|
|
1309
|
-
"looker.explore.file": self.source_file,
|
|
1310
|
-
}
|
|
1311
|
-
dataset_props = DatasetPropertiesClass(
|
|
1312
|
-
name=str(self.label) if self.label else LookerUtil._display_name(self.name),
|
|
1313
|
-
description=self.description,
|
|
1314
|
-
customProperties={
|
|
1315
|
-
k: str(v) for k, v in custom_properties.items() if v is not None
|
|
1316
|
-
},
|
|
1317
|
-
)
|
|
1318
|
-
dataset_props.externalUrl = self._get_url(base_url)
|
|
1306
|
+
) -> Dataset:
|
|
1307
|
+
"""
|
|
1308
|
+
Generate a Dataset metadata event for this Looker Explore.
|
|
1319
1309
|
|
|
1320
|
-
|
|
1310
|
+
Only generates datasets for explores that contain FROM clauses and do NOT contain joins.
|
|
1311
|
+
Passthrough explores and joins are handled via lineage and do not need additional nodes.
|
|
1312
|
+
"""
|
|
1313
|
+
upstream_lineage = None
|
|
1321
1314
|
view_name_to_urn_map: Dict[str, str] = {}
|
|
1315
|
+
|
|
1322
1316
|
if self.upstream_views is not None:
|
|
1323
1317
|
assert self.project_name is not None
|
|
1324
|
-
upstreams = []
|
|
1318
|
+
upstreams: list[UpstreamClass] = []
|
|
1325
1319
|
observed_lineage_ts = datetime.datetime.now(tz=datetime.timezone.utc)
|
|
1320
|
+
|
|
1326
1321
|
for view_ref in sorted(self.upstream_views):
|
|
1327
1322
|
# set file_path to ViewFieldType.UNKNOWN if file_path is not available to keep backward compatibility
|
|
1328
1323
|
# if we raise error on file_path equal to None then existing test-cases will fail as mock data
|
|
1329
1324
|
# doesn't have required attributes.
|
|
1330
1325
|
file_path: str = (
|
|
1331
1326
|
cast(str, self.upstream_views_file_path[view_ref.include])
|
|
1332
|
-
if self.upstream_views_file_path
|
|
1327
|
+
if self.upstream_views_file_path.get(view_ref.include) is not None
|
|
1333
1328
|
else ViewFieldValue.NOT_AVAILABLE.value
|
|
1334
1329
|
)
|
|
1335
1330
|
|
|
@@ -1356,7 +1351,7 @@ class LookerExplore:
|
|
|
1356
1351
|
)
|
|
1357
1352
|
view_name_to_urn_map[view_ref.include] = view_urn
|
|
1358
1353
|
|
|
1359
|
-
fine_grained_lineages = []
|
|
1354
|
+
fine_grained_lineages: list[FineGrainedLineageClass] = []
|
|
1360
1355
|
if config.extract_column_level_lineage:
|
|
1361
1356
|
for field in self.fields or []:
|
|
1362
1357
|
# Skip creating fine-grained lineage for empty field names to prevent invalid schema field URNs
|
|
@@ -1397,9 +1392,11 @@ class LookerExplore:
|
|
|
1397
1392
|
)
|
|
1398
1393
|
|
|
1399
1394
|
upstream_lineage = UpstreamLineage(
|
|
1400
|
-
upstreams=upstreams,
|
|
1395
|
+
upstreams=upstreams,
|
|
1396
|
+
fineGrainedLineages=fine_grained_lineages or None,
|
|
1401
1397
|
)
|
|
1402
|
-
|
|
1398
|
+
|
|
1399
|
+
schema_metadata = None
|
|
1403
1400
|
if self.fields is not None:
|
|
1404
1401
|
schema_metadata = LookerUtil._get_schema(
|
|
1405
1402
|
platform_name=config.platform_name,
|
|
@@ -1407,42 +1404,46 @@ class LookerExplore:
|
|
|
1407
1404
|
view_fields=self.fields,
|
|
1408
1405
|
reporter=reporter,
|
|
1409
1406
|
)
|
|
1410
|
-
if schema_metadata is not None:
|
|
1411
|
-
dataset_snapshot.aspects.append(schema_metadata)
|
|
1412
|
-
|
|
1413
|
-
mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
|
|
1414
|
-
mcp = MetadataChangeProposalWrapper(
|
|
1415
|
-
entityUrn=dataset_snapshot.urn,
|
|
1416
|
-
aspect=SubTypesClass(typeNames=[DatasetSubTypes.LOOKER_EXPLORE]),
|
|
1417
|
-
)
|
|
1418
|
-
|
|
1419
|
-
proposals: List[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]] = [
|
|
1420
|
-
mce,
|
|
1421
|
-
mcp,
|
|
1422
|
-
]
|
|
1423
1407
|
|
|
1424
|
-
|
|
1425
|
-
explore_tag_urns: List[TagAssociationClass] = [
|
|
1426
|
-
TagAssociationClass(tag=TagUrn(tag).urn()) for tag in self.tags
|
|
1427
|
-
]
|
|
1428
|
-
if explore_tag_urns:
|
|
1429
|
-
dataset_snapshot.aspects.append(GlobalTagsClass(explore_tag_urns))
|
|
1408
|
+
extra_aspects: List[Union[GlobalTagsClass, EmbedClass]] = []
|
|
1430
1409
|
|
|
1431
|
-
|
|
1410
|
+
explore_tag_urns: List[TagUrn] = [TagUrn(tag) for tag in self.tags]
|
|
1432
1411
|
if extract_embed_urls:
|
|
1433
|
-
|
|
1434
|
-
dataset_snapshot.urn, self._get_embed_url(base_url)
|
|
1435
|
-
)
|
|
1436
|
-
proposals.append(embed_mcp)
|
|
1412
|
+
extra_aspects.append(EmbedClass(renderUrl=self._get_embed_url(base_url)))
|
|
1437
1413
|
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1414
|
+
custom_properties: Dict[str, Optional[str]] = {
|
|
1415
|
+
"project": self.project_name,
|
|
1416
|
+
"model": self.model_name,
|
|
1417
|
+
"looker.explore.label": self.label,
|
|
1418
|
+
"looker.explore.name": self.name,
|
|
1419
|
+
"looker.explore.file": self.source_file,
|
|
1420
|
+
}
|
|
1444
1421
|
|
|
1445
|
-
return
|
|
1422
|
+
return Dataset(
|
|
1423
|
+
platform=config.platform_name,
|
|
1424
|
+
name=config.explore_naming_pattern.replace_variables(
|
|
1425
|
+
self.get_mapping(config)
|
|
1426
|
+
),
|
|
1427
|
+
display_name=str(self.label)
|
|
1428
|
+
if self.label
|
|
1429
|
+
else LookerUtil._display_name(self.name),
|
|
1430
|
+
description=self.description,
|
|
1431
|
+
subtype=DatasetSubTypes.LOOKER_EXPLORE,
|
|
1432
|
+
env=config.env,
|
|
1433
|
+
platform_instance=config.platform_instance,
|
|
1434
|
+
custom_properties={
|
|
1435
|
+
k: str(v) for k, v in custom_properties.items() if v is not None
|
|
1436
|
+
},
|
|
1437
|
+
external_url=self._get_url(base_url),
|
|
1438
|
+
upstreams=upstream_lineage,
|
|
1439
|
+
schema=schema_metadata,
|
|
1440
|
+
parent_container=[
|
|
1441
|
+
"Explore",
|
|
1442
|
+
gen_model_key(config, self.model_name).as_urn(),
|
|
1443
|
+
],
|
|
1444
|
+
tags=explore_tag_urns if explore_tag_urns else None,
|
|
1445
|
+
extra_aspects=extra_aspects,
|
|
1446
|
+
)
|
|
1446
1447
|
|
|
1447
1448
|
|
|
1448
1449
|
def gen_project_key(config: LookerCommonConfig, project_name: str) -> LookMLProjectKey:
|