acryl-datahub 1.2.0.10rc1__py3-none-any.whl → 1.2.0.10rc2__py3-none-any.whl

{acryl_datahub-1.2.0.10rc1.dist-info → acryl_datahub-1.2.0.10rc2.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.10rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.10rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=gbUD5kbOI-y4DH5wBU0PRgKeHjSTCNQ6fgZcMvqwDpg,324
+datahub/_version.py,sha256=EmO6ZWHg0m-jKxy3tyNs5CnPt86iFlMSrEkhzdbO54M,324
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -159,7 +159,7 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
 datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=0BwkpLhORbsiTHq0g_N_1cVVoZYdLR3qz02mNmsV9-M,4444
 datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/autogenerated/capability_summary.json,sha256=78zS8mOo54IkG9l5KJ_YBXkZmERSSo3pDIrefQx-PWA,110199
+datahub/ingestion/autogenerated/capability_summary.json,sha256=6n3XZj4xPZ3WVfQ29QQvqX5ancSXAqvjaMfpDqFQ8LI,110531
 datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
 datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
 datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -211,7 +211,7 @@ datahub/ingestion/source/demo_data.py,sha256=PbtCHlZx3wrKlOPPgkWhDQuPm7ZfIx2neXJ
 datahub/ingestion/source/elastic_search.py,sha256=2dwIcSbYMaq_RoSnxLGz4Q_20oJ8AGgMKunVIBIgYM8,23406
 datahub/ingestion/source/feast.py,sha256=rAqT7huVgi4c7iRU9qSbohPbNRrxZVw4PIvnfxNsiUk,18798
 datahub/ingestion/source/file.py,sha256=sHCWbtrQcXMMYPs_LUqofx0mk6IFN0G7Lyk9b0yRZMI,16082
-datahub/ingestion/source/ge_data_profiler.py,sha256=zG3LR_xjc4bz3WxtURWz-lE0rtHE8VjFmOOaPioGTnw,68755
+datahub/ingestion/source/ge_data_profiler.py,sha256=9lEQdLcMBa7znqa6Zz-QWA4Uiv8KiiCALMEERL37pgA,69318
 datahub/ingestion/source/ge_profiling_config.py,sha256=sG_0BwPDRG3I4PnhfWGHf9AbePLDWG0kKcKEtlXHTuk,11544
 datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suabri1yV5geaAPjzg2eORDc,2559
 datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
@@ -224,11 +224,11 @@ datahub/ingestion/source/openapi.py,sha256=1eemGG8BM5H8T2OxNSg6kzGDminblLPgPVuPA
 datahub/ingestion/source/openapi_parser.py,sha256=T87e2r-oPGgQl_FDMHnSGFZzApvWDCyKWnzIrVI5Alo,15420
 datahub/ingestion/source/preset.py,sha256=fncn-fgYcITsYEHVsvV6cGTQ9_xc_R06ejrw6ZbY3QA,3966
 datahub/ingestion/source/pulsar.py,sha256=u5F8QnCLJsht5-7XCiUTsnfhCPIpKVB_l32CgMCU-As,20187
-datahub/ingestion/source/redash.py,sha256=YxjSad-X_wPmxYH8dJmFz_VCFhiLTCTSlK99WdvcYiA,30653
+datahub/ingestion/source/redash.py,sha256=C4cDikWymbL88fDqaIPX5WA3f2sIEtH7bmhJKkmXJsM,30652
 datahub/ingestion/source/salesforce.py,sha256=UttN3y4Ylbx_yCFCr-33wUEZFR48nTiYeUfOjGIFj2E,40872
 datahub/ingestion/source/source_registry.py,sha256=a2mLjJPLkSI-gYCTb_7U7Jo4D8jGknNQ_yScPIihXFk,1208
 datahub/ingestion/source/sql_queries.py,sha256=Zf6Y84WFCD-j0v4HdtcXshAPrtGq5InncXCjnLaAuS4,14657
-datahub/ingestion/source/superset.py,sha256=SLv6DqglPlFJwEskLOqEr-rfgCAb_2AXqWYupk8FqH4,53756
+datahub/ingestion/source/superset.py,sha256=OWllzS0D0K9zUUQE6w-ZzemJduu6ZV003rBVe2rxkaM,56814
 datahub/ingestion/source/abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/abs/config.py,sha256=WW9JWbzqAJDblAcJKtNeuBHqOeJsB57lW2PqSD65-BU,6729
 datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=FfrcgK-JEF94vw-l3q6pN6FENXb-wZzW2w1VUZVkwW8,3620
@@ -302,7 +302,7 @@ datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vB
 datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
 datahub/ingestion/source/dbt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/dbt/dbt_cloud.py,sha256=ILkP6ui48VU8pClDQz6jaFR026gcF7lyUX4gt9im8Vg,18428
-datahub/ingestion/source/dbt/dbt_common.py,sha256=0JMrN4Gy33_KgG4UY0NQtIS6WIF_MnTm8LZ_fjr5v8I,89475
+datahub/ingestion/source/dbt/dbt_common.py,sha256=OGe9_tDMYuaM_9JZQ3fYtWkao09R5CB5x2SQlD1UDng,91881
 datahub/ingestion/source/dbt/dbt_core.py,sha256=WVI2ZYXOMxgFzJnJqsqmEGS-5xdfiVIDsCb78lvSeQ0,24930
 datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpidfGTvOi5fr6I,9839
 datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -383,7 +383,7 @@ datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfIC
 datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
 datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
-datahub/ingestion/source/looker/looker_common.py,sha256=7o2ZdL1u9d6yHfQt3v0abCouCE_SdDGuBeNDevcAqhY,67884
+datahub/ingestion/source/looker/looker_common.py,sha256=LKjGnPOKiWLD0cq-6eaFgXvIjzMdp-RPacwk_wMrXxA,68726
 datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
 datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
 datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
@@ -399,7 +399,7 @@ datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z
 datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
 datahub/ingestion/source/looker/lookml_config.py,sha256=lulLcjAS1d8ihQseBe4HYn6ALKmJX1vl0H5mxiBzZ74,11395
 datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
-datahub/ingestion/source/looker/lookml_source.py,sha256=iqvem_ND7w-ANcKKSjixM64jRIFOBZX-mNjYMZ_UDxU,44438
+datahub/ingestion/source/looker/lookml_source.py,sha256=aNvEoW0njzqQSoj1KeHiDOBbEGYWWDEcjX-a2RYgqOY,42735
 datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
 datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
 datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
@@ -635,8 +635,8 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_internal_schema_classes.py,sha256=C_RqtHu-dWkZUJWo85PNu9gs4evsVb4vMIbdAyRUM2o,1062642
-datahub/metadata/schema.avsc,sha256=Z5bziVmjWxly-mPLjCWic2eXMCkVIYTbWqRFErSK5uk,705606
+datahub/metadata/_internal_schema_classes.py,sha256=gijo0J9PIU9wETlfly4ngC1ognj9LyoFSXyiQCoWdzk,1068035
+datahub/metadata/schema.avsc,sha256=H39OyQNCy031TBDFbaa5ZGWiSYde8RZ5Q92GmhZ3F0M,708737
 datahub/metadata/schema_classes.py,sha256=tPT8iHCak4IsZi_oL0nirbPpI8ETTPTZzapqLRpeKU4,1326
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
@@ -709,7 +709,7 @@ datahub/metadata/com/linkedin/pegasus2avro/schemafield/__init__.py,sha256=HTWezn
 datahub/metadata/com/linkedin/pegasus2avro/secret/__init__.py,sha256=qk61EqqVZF6k1Ct6t4Uo-pLb0WtM1EwJKn1XjVy9LHE,305
 datahub/metadata/com/linkedin/pegasus2avro/settings/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py,sha256=ASD1mi7q19HVN9QKzbu2T3GxzdDQ_LqZNTehV3SI12c,531
-datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=hKpTDXj2YEBja4hk2fZx3G9yzJ7eCFxcGCbbN0NydTk,933
+datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py,sha256=kTY87JYIsV2yvBiHE-QrzPc7PIrUFS5aWh_VFQm7tSg,1107
 datahub/metadata/com/linkedin/pegasus2avro/step/__init__.py,sha256=HLNNbqBlyhcg09eXWx_AMD_JoOtBPYEi2kv12PE0R9E,329
 datahub/metadata/com/linkedin/pegasus2avro/structured/__init__.py,sha256=Cry61gPw6m5MQuJpPxADRm3jhI0XVqzznyD3fVKMkvc,1013
 datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py,sha256=Odb4mzloKJIlpoFHODEIxt_OIgFNrZExcyQtvXxjOFQ,290
@@ -857,7 +857,7 @@ datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8
 datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
 datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
 datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
-datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=HKZisxW2HNm6kwJUBaAVol7suJoxSVCjfvWy55HhalQ,11999
+datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=xxwH79mYno0nDt-TZ1bTlEd_8Tv4t01iijYHgPl9838,14927
 datahub/metadata/schemas/GlobalSettingsKey.avsc,sha256=Yj8s5IdM9yF7xrhJcLGCPCXBWqSsrPbufBaQjlZ3JlU,563
 datahub/metadata/schemas/GlobalTags.avsc,sha256=-SurkodMqTDnPpkRV6qYqmpNWjQNvynUiPZX7EhL5uc,4624
 datahub/metadata/schemas/GlossaryNodeInfo.avsc,sha256=G1Cb-w9VxIAEhNqyiEsDL_ABRO9QxyTpUANKU6DQrFw,1888
@@ -878,7 +878,7 @@ datahub/metadata/schemas/InstitutionalMemory.avsc,sha256=0dKK18Gr6eBIgUimZrm9zsB
 datahub/metadata/schemas/IntendedUse.avsc,sha256=IKZSWdvc0uAyyT-FtdQOGbMC-P7RS9cO0vOVKWT6fbw,1361
 datahub/metadata/schemas/InviteToken.avsc,sha256=8k_9MxHu9GVf7gvS0SlnQu7tJfpbXsRFdz6lQrFKPNc,737
 datahub/metadata/schemas/InviteTokenKey.avsc,sha256=MuQUlQaeVjaBkjSshB9gsx5Fm0civYgWD8UhCiRLdOQ,434
-datahub/metadata/schemas/LogicalParent.avsc,sha256=ocwb2ED-MNNRIPcFFjsFz5Qt7qHeVhFI5ou80zL5XuI,5420
+datahub/metadata/schemas/LogicalParent.avsc,sha256=Wr58mqOVKNc2YIXZSWEjIHqfcJ_U0XH1CRMuMAD5R_Q,5477
 datahub/metadata/schemas/MLFeatureKey.avsc,sha256=1XFF8P2T3_4mX2oHGY74UJx1LFr6WtTv2jTG9ApjQoA,1150
 datahub/metadata/schemas/MLFeatureProperties.avsc,sha256=HpF7VcnH2FvDsqy2g5AMJPqU7upkTVADW1ps-9bjINo,6893
 datahub/metadata/schemas/MLFeatureTableKey.avsc,sha256=hVxNEqsx4GgG11GVryn9ms16OyEzsYdwM5RuAHy_454,1424
@@ -1114,8 +1114,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.10rc1.dist-info/METADATA,sha256=RZTwZLgqey2EHL-O4LEZfy5CEJc5WYKTwMzdOl6aoFU,186878
-acryl_datahub-1.2.0.10rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.10rc1.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
-acryl_datahub-1.2.0.10rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.10rc1.dist-info/RECORD,,
+acryl_datahub-1.2.0.10rc2.dist-info/METADATA,sha256=bPXVHE45T0nHRzSOr4cTIhKQVXcO7TfksCB9yifFx0k,186878
+acryl_datahub-1.2.0.10rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.10rc2.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
+acryl_datahub-1.2.0.10rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.10rc2.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.10rc1"
+__version__ = "1.2.0.10rc2"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/autogenerated/capability_summary.json

@@ -2372,6 +2372,12 @@
           "subtype_modifier": null,
           "supported": true
         },
+        {
+          "capability": "TAGS",
+          "description": "Supported by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
         {
           "capability": "LINEAGE_COARSE",
           "description": "Supported by default",
@@ -3174,6 +3180,12 @@
           "subtype_modifier": null,
           "supported": true
         },
+        {
+          "capability": "TAGS",
+          "description": "Supported by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
         {
           "capability": "LINEAGE_COARSE",
           "description": "Supported by default",

datahub/ingestion/source/dbt/dbt_common.py

@@ -246,6 +246,23 @@ class DBTEntitiesEnabled(ConfigModel):
         return self.model_performance == EmitDirective.YES
 
 
+class MaterializedNodePatternConfig(ConfigModel):
+    """Configuration for filtering materialized nodes based on their physical location"""
+
+    database_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for database names to filter materialized nodes.",
+    )
+    schema_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for schema names in format '{database}.{schema}' to filter materialized nodes.",
+    )
+    table_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex patterns for table/view names in format '{database}.{schema}.{table}' to filter materialized nodes.",
+    )
+
+
 class DBTCommonConfig(
     StatefulIngestionConfigBase,
     PlatformInstanceConfigMixin,
@@ -294,6 +311,11 @@ class DBTCommonConfig(
         default=AllowDenyPattern.allow_all(),
         description="regex patterns for dbt model names to filter in ingestion.",
     )
+    materialized_node_pattern: MaterializedNodePatternConfig = Field(
+        default=MaterializedNodePatternConfig(),
+        description="Advanced filtering for materialized nodes based on their physical database location. "
+        "Provides fine-grained control over database.schema.table patterns for catalog consistency.",
+    )
     meta_mapping: Dict = Field(
         default={},
         description="mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.",
@@ -1018,15 +1040,53 @@ class DBTSourceBase(StatefulIngestionSourceBase):
             all_nodes_map,
         )
 
-    def _is_allowed_node(self, key: str) -> bool:
-        return self.config.node_name_pattern.allowed(key)
+    def _is_allowed_node(self, node: DBTNode) -> bool:
+        """
+        Check whether a node should be processed, using multi-layer rules. Checks for materialized nodes might need to be restricted in the future to some cases
+        """
+        if not self.config.node_name_pattern.allowed(node.dbt_name):
+            return False
+
+        if not self._is_allowed_materialized_node(node):
+            return False
+
+        return True
+
+    def _is_allowed_materialized_node(self, node: DBTNode) -> bool:
+        """Filter nodes based on their materialized database location for catalog consistency"""
+
+        # Database level filtering
+        if not node.database:
+            return True
+        if not self.config.materialized_node_pattern.database_pattern.allowed(
+            node.database
+        ):
+            return False
+
+        # Schema level filtering: {database}.{schema}
+        if not node.schema:
+            return True
+        if not self.config.materialized_node_pattern.schema_pattern.allowed(
+            node._join_parts([node.database, node.schema])
+        ):
+            return False
+
+        # Table level filtering: {database}.{schema}.{table}
+        if not node.name:
+            return True
+        if not self.config.materialized_node_pattern.table_pattern.allowed(
+            node.get_db_fqn()
+        ):
+            return False
+
+        return True
 
     def _filter_nodes(self, all_nodes: List[DBTNode]) -> List[DBTNode]:
         nodes: List[DBTNode] = []
         for node in all_nodes:
             key = node.dbt_name
 
-            if not self._is_allowed_node(key):
+            if not self._is_allowed_node(node):
                 self.report.nodes_filtered.append(key)
                 continue
 
@@ -1118,8 +1178,8 @@ class DBTSourceBase(StatefulIngestionSourceBase):
             cll_nodes.add(dbt_name)
             schema_nodes.add(dbt_name)
 
-        for dbt_name in all_nodes_map:
-            if self._is_allowed_node(dbt_name):
+        for dbt_name, dbt_node in all_nodes_map.items():
+            if self._is_allowed_node(dbt_node):
                 add_node_to_cll_list(dbt_name)
 
         return schema_nodes, cll_nodes

datahub/ingestion/source/ge_data_profiler.py

@@ -307,7 +307,6 @@ def _is_single_row_query_method(query: Any) -> bool:
         "get_column_max",
         "get_column_mean",
         "get_column_stdev",
-        "get_column_nonnull_count",
         "get_column_unique_count",
     }
     CONSTANT_ROW_QUERY_METHODS = {
@@ -331,6 +330,7 @@ def _is_single_row_query_method(query: Any) -> bool:
 
     FIRST_PARTY_SINGLE_ROW_QUERY_METHODS = {
         "get_column_unique_count_dh_patch",
+        "_get_column_cardinality",
     }
 
     # We'll do this the inefficient way since the arrays are pretty small.
@@ -497,7 +497,20 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase):
         self, column_spec: _SingleColumnSpec, column: str
     ) -> None:
         try:
-            nonnull_count = self.dataset.get_column_nonnull_count(column)
+            # Don't use Great Expectations get_column_nonnull_count because it
+            # generates this SQL:
+            #
+            #   sum(CASE WHEN (mycolumn IN (NULL) OR mycolumn IS NULL) THEN 1 ELSE 0 END)
+            #
+            # which fails for complex types (such as Databricks maps) that don't
+            # support the IN operator.
+            nonnull_count = convert_to_json_serializable(
+                self.dataset.engine.execute(
+                    sa.select(sa.func.count(sa.column(column))).select_from(
+                        self.dataset._table
+                    )
+                ).scalar()
+            )
             column_spec.nonnull_count = nonnull_count
         except Exception as e:
             logger.debug(

datahub/ingestion/source/looker/looker_common.py

@@ -255,6 +255,11 @@ class LookerViewId:
 
         return generated_urn
 
+    def get_view_dataset_name(self, config: LookerCommonConfig) -> str:
+        n_mapping: ViewNamingPatternMapping = self.get_mapping(config)
+        n_mapping.file_path = self.preprocess_file_path(n_mapping.file_path)
+        return config.view_naming_pattern.replace_variables(n_mapping)
+
     def get_browse_path(self, config: LookerCommonConfig) -> str:
         browse_path = config.view_browse_pattern.replace_variables(
             self.get_mapping(config)
@@ -282,6 +287,22 @@ class LookerViewId:
             ],
         )
 
+    def get_view_dataset_parent_container(
+        self, config: LookerCommonConfig
+    ) -> List[str]:
+        project_key = gen_project_key(config, self.project_name)
+        view_path = (
+            remove_suffix(self.file_path, ".view.lkml")
+            if "{file_path}" in config.view_browse_pattern.pattern
+            else os.path.dirname(self.file_path)
+        )
+        path_entries = view_path.split("/") if view_path else []
+        return [
+            "Develop",
+            project_key.as_urn(),
+            *path_entries,
+        ]
+
 
 class ViewFieldType(Enum):
     DIMENSION = "Dimension"

datahub/ingestion/source/looker/lookml_source.py

@@ -4,7 +4,7 @@ import tempfile
 from collections import OrderedDict
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Dict, Iterable, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import lkml
 import lkml.simple
@@ -12,8 +12,7 @@ from looker_sdk.error import SDKError
 
 from datahub.configuration.git import GitInfo
 from datahub.emitter.mce_builder import make_schema_field_urn
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import gen_containers
+from datahub.emitter.mcp_builder import mcps_from_mce
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -77,7 +76,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status
+from datahub.metadata.com.linkedin.pegasus2avro.common import Status
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageTypeClass,
     FineGrainedLineageDownstreamType,
@@ -85,18 +84,15 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     UpstreamLineage,
     ViewProperties,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
-from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     AuditStampClass,
-    BrowsePathEntryClass,
-    BrowsePathsV2Class,
-    ContainerClass,
     DatasetPropertiesClass,
     FineGrainedLineageClass,
     FineGrainedLineageUpstreamTypeClass,
-    SubTypesClass,
 )
+from datahub.sdk.container import Container
+from datahub.sdk.dataset import Dataset
+from datahub.sdk.entity import Entity
 from datahub.sql_parsing.sqlglot_lineage import ColumnRef
 
 VIEW_LANGUAGE_LOOKML: str = "lookml"
@@ -428,69 +424,40 @@ class LookMLSource(StatefulIngestionSourceBase):
 
         return dataset_props
 
-    def _build_dataset_mcps(
-        self, looker_view: LookerView
-    ) -> List[MetadataChangeProposalWrapper]:
-        view_urn = looker_view.id.get_urn(self.source_config)
-
-        subTypeEvent = MetadataChangeProposalWrapper(
-            entityUrn=view_urn,
-            aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW]),
-        )
-        events = [subTypeEvent]
+    def _build_dataset_entities(self, looker_view: LookerView) -> Iterable[Dataset]:
+        dataset_extra_aspects: List[Union[ViewProperties, Status]] = [
+            Status(removed=False)
+        ]
         if looker_view.view_details is not None:
-            viewEvent = MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.view_details,
-            )
-            events.append(viewEvent)
-
-        project_key = gen_project_key(self.source_config, looker_view.id.project_name)
-
-        container = ContainerClass(container=project_key.as_urn())
-        events.append(
-            MetadataChangeProposalWrapper(entityUrn=view_urn, aspect=container)
-        )
-
-        events.append(
-            MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.id.get_browse_path_v2(self.source_config),
-            )
-        )
-
-        return events
-
-    def _build_dataset_mce(self, looker_view: LookerView) -> MetadataChangeEvent:
-        """
-        Creates MetadataChangeEvent for the dataset, creating upstream lineage links
-        """
-        logger.debug(f"looker_view = {looker_view.id}")
+            dataset_extra_aspects.append(looker_view.view_details)
 
-        dataset_snapshot = DatasetSnapshot(
-            urn=looker_view.id.get_urn(self.source_config),
-            aspects=[],  # we append to this list later on
-        )
-        browse_paths = BrowsePaths(
-            paths=[looker_view.id.get_browse_path(self.source_config)]
-        )
-
-        dataset_snapshot.aspects.append(browse_paths)
-        dataset_snapshot.aspects.append(Status(removed=False))
-        upstream_lineage = self._get_upstream_lineage(looker_view)
-        if upstream_lineage is not None:
-            dataset_snapshot.aspects.append(upstream_lineage)
         schema_metadata = LookerUtil._get_schema(
             self.source_config.platform_name,
             looker_view.id.view_name,
             looker_view.fields,
             self.reporter,
         )
-        if schema_metadata is not None:
-            dataset_snapshot.aspects.append(schema_metadata)
-        dataset_snapshot.aspects.append(self._get_custom_properties(looker_view))
 
-        return MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+        custom_properties: DatasetPropertiesClass = self._get_custom_properties(
+            looker_view
+        )
+
+        yield Dataset(
+            platform=self.source_config.platform_name,
+            name=looker_view.id.get_view_dataset_name(self.source_config),
+            display_name=looker_view.id.view_name,
+            platform_instance=self.source_config.platform_instance,
+            env=self.source_config.env,
+            subtype=DatasetSubTypes.VIEW,
+            parent_container=looker_view.id.get_view_dataset_parent_container(
+                self.source_config
+            ),
+            schema=schema_metadata,
+            custom_properties=custom_properties.customProperties,
+            external_url=custom_properties.externalUrl,
+            upstreams=self._get_upstream_lineage(looker_view),
+            extra_aspects=dataset_extra_aspects,
+        )
 
     def get_project_name(self, model_name: str) -> str:
         if self.source_config.project_name is not None:
@@ -554,7 +521,7 @@ class LookMLSource(StatefulIngestionSourceBase):
             ).workunit_processor,
         ]
 
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         with tempfile.TemporaryDirectory("lookml_tmp") as tmp_dir:
             # Clone the base_folder if necessary.
             if not self.source_config.base_folder:
@@ -715,7 +682,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                 tmp_dir, project, project_visited, manifest_constants
             )
 
-    def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+    def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:  # noqa: C901
         assert self.source_config.base_folder
         viewfile_loader = LookerViewFileLoader(
             self.source_config.project_name,
@@ -949,7 +916,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                                         maybe_looker_view.id.project_name
                                         not in self.processed_projects
                                     ):
-                                        yield from self.gen_project_workunits(
+                                        yield from self.gen_project_containers(
                                             maybe_looker_view.id.project_name
                                         )
 
@@ -957,15 +924,10 @@ class LookMLSource(StatefulIngestionSourceBase):
                                             maybe_looker_view.id.project_name
                                         )
 
-                                    for mcp in self._build_dataset_mcps(
+                                    yield from self._build_dataset_entities(
                                         maybe_looker_view
-                                    ):
-                                        yield mcp.as_workunit()
-                                    mce = self._build_dataset_mce(maybe_looker_view)
-                                    yield MetadataWorkUnit(
-                                        id=f"lookml-view-{maybe_looker_view.id}",
-                                        mce=mce,
                                     )
+
                                     processed_view_files.add(include.include)
                                 else:
                                     (
@@ -994,28 +956,24 @@ class LookMLSource(StatefulIngestionSourceBase):
             self.source_config.tag_measures_and_dimensions
             and self.reporter.events_produced != 0
         ):
-            # Emit tag MCEs for measures and dimensions:
+            # Emit tag MCEs for measures and dimensions if we produced any explores:
             for tag_mce in LookerUtil.get_tag_mces():
-                yield MetadataWorkUnit(
-                    id=f"tag-{tag_mce.proposedSnapshot.urn}", mce=tag_mce
-                )
+                # Convert MCE to MCPs
+                for mcp in mcps_from_mce(tag_mce):
+                    yield mcp.as_workunit()
 
-    def gen_project_workunits(self, project_name: str) -> Iterable[MetadataWorkUnit]:
+    def gen_project_containers(self, project_name: str) -> Iterable[Container]:
         project_key = gen_project_key(
             self.source_config,
             project_name,
         )
-        yield from gen_containers(
+
+        yield Container(
             container_key=project_key,
-            name=project_name,
-            sub_types=[BIContainerSubTypes.LOOKML_PROJECT],
+            display_name=project_name,
+            subtype=BIContainerSubTypes.LOOKML_PROJECT,
+            parent_container=["Folders"],
         )
-        yield MetadataChangeProposalWrapper(
-            entityUrn=project_key.as_urn(),
-            aspect=BrowsePathsV2Class(
-                path=[BrowsePathEntryClass("Folders")],
-            ),
-        ).as_workunit()
 
     def report_skipped_unreachable_views(
         self,

datahub/ingestion/source/redash.py

@@ -447,7 +447,7 @@ class RedashSource(StatefulIngestionSourceBase):
                 dataset_urns = sql_parser_in_tables.in_tables
                 if sql_parser_in_tables.debug_info.table_error:
                     self.report.queries_problem_parsing.add(str(query_id))
-                    self.error(
+                    self.warn(
                         logger,
                         "sql-parsing",
                         f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",