acryl-datahub 0.15.0.5rc4__py3-none-any.whl → 0.15.0.5rc6__py3-none-any.whl

{acryl_datahub-0.15.0.5rc4.dist-info → acryl_datahub-0.15.0.5rc6.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=WfLwqZVZ8jnBAUkc1mxyo_REl_AjCDcwm9R97SuldaY,324
+datahub/_version.py,sha256=w4julQ8A-97paLIXUIUhvcioGTJP8CcU4tW-IaJw3zY,324
 datahub/entrypoints.py,sha256=osv2ailvuW-HHlAE0fOtyblJI1X7HInZutd9DC66jqQ,8022
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/_codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -62,7 +62,7 @@ datahub/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/cli/check_cli.py,sha256=ajrWVMAHYbgvYi4OFitFXx7Y6oigvZFgIeUiKV9ECik,12859
 datahub/cli/cli_utils.py,sha256=2uvPv6WqxbRdH7UteHwhRash4E0ncU5P6XebrFLeECo,13584
 datahub/cli/config_utils.py,sha256=yuXw7RzpRY5x_-MAoqWbv46qUkIeRNAJL4_OeJpYdBE,4879
-datahub/cli/container_cli.py,sha256=8D73hLfTHsDg4Cedh_2x0utl7ppOeB1TUJVRgur-Crw,3624
+datahub/cli/container_cli.py,sha256=uDOwewGEPYHQt-ppYEb8ESXhZjPNIZG0Rt3cm2FzPqc,1569
 datahub/cli/delete_cli.py,sha256=oQ4Yy6hxZHcl67MYJiQumLs_8QmFEj7SPZFzxFXvDk8,23481
 datahub/cli/docker_check.py,sha256=rED4wHXqxcQ_qNFyIgFEZ85BHT9ZTE5YC-oUKqbRqi0,9432
 datahub/cli/docker_cli.py,sha256=w9ZQMRVlHwfJI2XDe7mO0lwnT7-dZoK6tPadSMgwEM8,36493
@@ -113,6 +113,7 @@ datahub/configuration/validate_multiline_string.py,sha256=l9PF6_EAC_1lWxU_RWrvPB
 datahub/configuration/yaml.py,sha256=dLmjCalPOjgdc7mmJxtlP7uOrIHZiAWxD1gwAFOdtUU,308
 datahub/emitter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/emitter/aspect.py,sha256=ef0DVycqg-tRPurkYjc-5zknmLP2p2Y2RxP55WkvAEc,480
+datahub/emitter/composite_emitter.py,sha256=ZU-IdlAXKGPtmyT0JJgYC09vRn-TmeNaA6VP8V0fioM,1212
 datahub/emitter/enum_helpers.py,sha256=QBOEUu_hDCvyL_v4ayNQV8XwJbf5zKyu0Xat0mI1Kgo,376
 datahub/emitter/generic_emitter.py,sha256=i37ZFm9VR_tmiZm9kIypEkQEB_cLKbzj_tJvViN-fm8,828
 datahub/emitter/kafka_emitter.py,sha256=Uix1W1WaXF8VqUTUfzdRZKca2XrR1w50Anx2LVkROlc,5822
@@ -217,6 +218,8 @@ datahub/ingestion/source/abs/datalake_profiler_config.py,sha256=Rkf64evufyVGPiE4
 datahub/ingestion/source/abs/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
 datahub/ingestion/source/abs/report.py,sha256=fzkTdTewYlWrTk4f2Cyl-e8RV4qw9wEVtm0cdKD-Xgw,542
 datahub/ingestion/source/abs/source.py,sha256=cuMezUzr-Smp5tok2ceYor5I5jp52NDMjfeN8kfIbvg,24816
+datahub/ingestion/source/apply/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub/ingestion/source/apply/datahub_apply.py,sha256=NGhbszi6ObfJoh3YPGjcVWHJsbNboVwKUgnrCSt8HJU,7629
 datahub/ingestion/source/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/aws/aws_common.py,sha256=DfdQgkJ_s2isFx8WvqKTlAcBk4KE8SgfpmA5BgC3fgY,17716
 datahub/ingestion/source/aws/glue.py,sha256=DwROr923M01QnvImUbMoHS6TTTT9kBz2tEmQ3Sv4EoY,58019
@@ -333,22 +336,22 @@ datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=-rFNXKD8_EFoX
 datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
 datahub/ingestion/source/looker/looker_common.py,sha256=squUUBHxsLeT5xbZOTO66irtOB8fL0V4Q8Tgd9EJMYU,62067
-datahub/ingestion/source/looker/looker_config.py,sha256=Kp-IoGcjfk5yPoqXFbsjo6aLA1j0H9LFEzZwhoGNvy4,13584
+datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
 datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
 datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
-datahub/ingestion/source/looker/looker_dataclasses.py,sha256=LjrP5m_A4HV-XeFlSNGVYNuyF0ulxp_qwB82Ss4Iycs,12200
-datahub/ingestion/source/looker/looker_file_loader.py,sha256=c1ewDrIb9VJg1o-asbwX9gL83kgL01vIETzzbmZIhmw,4267
+datahub/ingestion/source/looker/looker_dataclasses.py,sha256=MrDeZ4Nd0wQnJbCoI1qePYlYeObnUw5dvpWcmhKuNgc,12346
+datahub/ingestion/source/looker/looker_file_loader.py,sha256=PEyL9KWRaFcrvOkapU8wSNlFbmetmBy9tAyCgeVDOa4,4864
 datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=0gaYjBv4wkbbLWVgvaAV6JyWAFb0utTG6TCve2d9xss,11511
 datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=mO4G4MNA4YZFvZaDBpdiJ2vP3irC82kY34RdaK4Pbfs,3100
 datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
 datahub/ingestion/source/looker/looker_source.py,sha256=S-g06Bm3sbyD0Qjra9hEhZmsVDb-BY_-bCPDwCjtEoQ,66427
-datahub/ingestion/source/looker/looker_template_language.py,sha256=mfbU27NYs0mkZHXdtvS38FC5WCJ4S_aGjC8t09yecKY,14330
+datahub/ingestion/source/looker/looker_template_language.py,sha256=W-SMICKBfIuivrHywHRYchz9SJiXhoU8VOEKGQW_1v8,17825
 datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
 datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
 datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
-datahub/ingestion/source/looker/lookml_config.py,sha256=Ub5Efgzb1bDId5nNcUhcZKEm2hp273wF5edip283U2g,10775
+datahub/ingestion/source/looker/lookml_config.py,sha256=RuZkH3DDmII21gEsUvPsJi5gxWngbYkqBP06H8_n_Hs,11353
 datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
-datahub/ingestion/source/looker/lookml_source.py,sha256=qkLVzs5AHSA5B518s7fCD99KKHh6_8-QR8KM0mySnFg,42757
+datahub/ingestion/source/looker/lookml_source.py,sha256=PJBUJgZfZyvmasDf_LJC39SggLCA6vSfAbf1PdzviZU,43889
 datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
 datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
 datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
@@ -446,7 +449,7 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYh
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=TN_cTF4a8V_tbeR2czm_hoMYfQMuqmBbFlAqyh7PJzQ,6551
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
 datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=xq58c47zmaQPkTVqjKW25iViX8VJuHdQDTFY4jxzZ2o,12778
-datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=8nnQ_XlbT4q6y6_JleSV67njUaPjF9x1yehKu5I1lgc,32072
+datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=vyTqC_C5Bf0AMRVyoxUfl1CdlgeQouX20msP2FsMqnk,33439
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=Uh9wGLOqAkcphffxOPIQNyXvjeRm74XIpaLb4rjqMjM,24045
 datahub/ingestion/source/sql/clickhouse.py,sha256=uSRy-HKAiGFTHVLoVtGoh23X0O1lwyYUaK8BaWkYhps,25555
@@ -993,9 +996,9 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-0.15.0.5rc4.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
-acryl_datahub-0.15.0.5rc4.dist-info/METADATA,sha256=5_fFeSDo0RY3z5NeouX22pyoKVy8iM7I7a8KyYQz4Xg,173382
-acryl_datahub-0.15.0.5rc4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-acryl_datahub-0.15.0.5rc4.dist-info/entry_points.txt,sha256=xnPSPLK3bJGADxe4TDS4wL4u0FT_PGlahDa-ENYdYCQ,9512
-acryl_datahub-0.15.0.5rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-0.15.0.5rc4.dist-info/RECORD,,
+acryl_datahub-0.15.0.5rc6.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-0.15.0.5rc6.dist-info/METADATA,sha256=EYc5u1TJYgxZjmer0MNvW4TpmUkIwDFwJ1qTmQsrkxQ,173382
+acryl_datahub-0.15.0.5rc6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+acryl_datahub-0.15.0.5rc6.dist-info/entry_points.txt,sha256=U1e5ZwqPX1OaIbvGrwvozcdB8SbzFYXQM7plpdLKKeo,9592
+acryl_datahub-0.15.0.5rc6.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-0.15.0.5rc6.dist-info/RECORD,,

{acryl_datahub-0.15.0.5rc4.dist-info → acryl_datahub-0.15.0.5rc6.dist-info}/entry_points.txt

@@ -34,6 +34,7 @@ clickhouse-usage = datahub.ingestion.source.usage.clickhouse_usage:ClickHouseUsa
 cockroachdb = datahub.ingestion.source.sql.cockroachdb:CockroachDBSource
 csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource
 datahub = datahub.ingestion.source.datahub.datahub_source:DataHubSource
+datahub-apply = datahub.ingestion.source.apply.datahub_apply:DataHubApplySource
 datahub-business-glossary = datahub.ingestion.source.metadata.business_glossary:BusinessGlossaryFileSource
 datahub-gc = datahub.ingestion.source.gc.datahub_gc:DataHubGcSource
 datahub-lineage-file = datahub.ingestion.source.metadata.lineage:LineageFileSource

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "0.15.0.5rc4"
+__version__ = "0.15.0.5rc6"
 
 
 def is_dev_mode() -> bool:

datahub/cli/container_cli.py

@@ -1,19 +1,8 @@
 import logging
-from typing import Any, List
 
 import click
-import progressbar
 
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.ingestion.graph.client import get_default_graph
-from datahub.metadata.schema_classes import (
-    DomainsClass,
-    GlossaryTermAssociationClass,
-    OwnerClass,
-    OwnershipTypeClass,
-    TagAssociationClass,
-)
-from datahub.specific.dataset import DatasetPatchBuilder
+from datahub.ingestion.source.apply.datahub_apply import apply_association_to_container
 
 logger = logging.getLogger(__name__)
 
@@ -24,58 +13,6 @@ def container() -> None:
     pass
 
 
-def apply_association_to_container(
-    container_urn: str,
-    association_urn: str,
-    association_type: str,
-) -> None:
-    """
-    Common function to add either tags, terms, domains, or owners to child datasets (for now).
-
-    Args:
-        container_urn: The URN of the container
-        association_urn: The URN of the tag, term, or user to apply
-        association_type: One of 'tag', 'term', 'domain' or 'owner'
-    """
-    urns: List[str] = []
-    graph = get_default_graph()
-    logger.info(f"Using {graph}")
-    urns.extend(
-        graph.get_urns_by_filter(
-            container=container_urn, batch_size=1000, entity_types=["dataset"]
-        )
-    )
-
-    all_patches: List[Any] = []
-    for urn in urns:
-        builder = DatasetPatchBuilder(urn)
-        patches: List[Any] = []
-        if association_type == "tag":
-            patches = builder.add_tag(TagAssociationClass(association_urn)).build()
-        elif association_type == "term":
-            patches = builder.add_term(
-                GlossaryTermAssociationClass(association_urn)
-            ).build()
-        elif association_type == "owner":
-            patches = builder.add_owner(
-                OwnerClass(
-                    owner=association_urn,
-                    type=OwnershipTypeClass.TECHNICAL_OWNER,
-                )
-            ).build()
-        elif association_type == "domain":
-            patches = [
-                MetadataChangeProposalWrapper(
-                    entityUrn=urn,
-                    aspect=DomainsClass(domains=[association_urn]),
-                )
-            ]
-        all_patches.extend(patches)
-    mcps_iter = progressbar.progressbar(all_patches, redirect_stdout=True)
-    for mcp in mcps_iter:
-        graph.emit(mcp)
-
-
 @container.command()
 @click.option("--container-urn", required=True, type=str)
 @click.option("--tag-urn", required=True, type=str)

datahub/emitter/composite_emitter.py

@@ -0,0 +1,36 @@
+from typing import Callable, List, Optional, Union
+
+from datahub.emitter.generic_emitter import Emitter
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.metadata.com.linkedin.pegasus2avro.mxe import (
+    MetadataChangeEvent,
+    MetadataChangeProposal,
+)
+
+
+# Experimental composite emitter that allows multiple emitters to be used in a single ingestion job
+class CompositeEmitter(Emitter):
+    def __init__(self, emitters: List[Emitter]) -> None:
+        self.emitters = emitters
+
+    def emit(
+        self,
+        item: Union[
+            MetadataChangeEvent,
+            MetadataChangeProposal,
+            MetadataChangeProposalWrapper,
+        ],
+        callback: Optional[Callable[[Exception, str], None]] = None,
+    ) -> None:
+        callback_called = False
+        for emitter in self.emitters:
+            if not callback_called:
+                # We want to ensure that the callback is only called once and we tie it to the first emitter
+                emitter.emit(item, callback)
+                callback_called = True
+            else:
+                emitter.emit(item)
+
+    def flush(self) -> None:
+        for emitter in self.emitters:
+            emitter.flush()

datahub/ingestion/source/apply/datahub_apply.py

@@ -0,0 +1,223 @@
+import logging
+from functools import partial
+from typing import Any, Iterable, List, Optional, Union
+
+import progressbar
+from pydantic import Field
+
+from datahub.configuration.common import ConfigModel
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source, SourceReport
+from datahub.ingestion.api.source_helpers import auto_workunit_reporter
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
+from datahub.metadata.schema_classes import (
+    DomainsClass,
+    GlossaryTermAssociationClass,
+    MetadataChangeProposalClass,
+    OwnerClass,
+    OwnershipTypeClass,
+    TagAssociationClass,
+)
+from datahub.specific.dataset import DatasetPatchBuilder
+
+logger = logging.getLogger(__name__)
+
+
+def apply_association_to_container(
+    container_urn: str,
+    association_urn: str,
+    association_type: str,
+    emit: bool = True,
+    graph: Optional[DataHubGraph] = None,
+) -> Optional[List[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]]:
+    """
+    Common function to add either tags, terms, domains, or owners to child datasets (for now).
+
+    Args:
+        container_urn: The URN of the container
+        association_urn: The URN of the tag, term, or user to apply
+        association_type: One of 'tag', 'term', 'domain' or 'owner'
+    """
+    urns: List[str] = [container_urn]
+    if not graph:
+        graph = get_default_graph()
+    logger.info(f"Using {graph}")
+    urns.extend(
+        graph.get_urns_by_filter(
+            container=container_urn,
+            batch_size=1000,
+            entity_types=["dataset", "container"],
+        )
+    )
+
+    all_patches: List[Any] = []
+    for urn in urns:
+        builder = DatasetPatchBuilder(urn)
+        patches: List[Any] = []
+        if association_type == "tag":
+            patches = builder.add_tag(TagAssociationClass(association_urn)).build()
+        elif association_type == "term":
+            patches = builder.add_term(
+                GlossaryTermAssociationClass(association_urn)
+            ).build()
+        elif association_type == "owner":
+            patches = builder.add_owner(
+                OwnerClass(
+                    owner=association_urn,
+                    type=OwnershipTypeClass.TECHNICAL_OWNER,
+                )
+            ).build()
+        elif association_type == "domain":
+            patches = [
+                MetadataChangeProposalWrapper(
+                    entityUrn=urn,
+                    aspect=DomainsClass(domains=[association_urn]),
+                )
+            ]
+        all_patches.extend(patches)
+    if emit:
+        mcps_iter = progressbar.progressbar(all_patches, redirect_stdout=True)
+        for mcp in mcps_iter:
+            graph.emit(mcp)
+        return None
+    else:
+        return all_patches
+
+
+class DomainApplyConfig(ConfigModel):
+    assets: List[str] = Field(
+        default_factory=list,
+        description="List of assets to apply domain hierarchichaly. Currently only containers and datasets are supported",
+    )
+    domain_urn: str = Field(default="")
+
+
+class TagApplyConfig(ConfigModel):
+    assets: List[str] = Field(
+        default_factory=list,
+        description="List of assets to apply tag hierarchichaly. Currently only containers and datasets are supported",
+    )
+    tag_urn: str = Field(default="")
+
+
+class TermApplyConfig(ConfigModel):
+    assets: List[str] = Field(
+        default_factory=list,
+        description="List of assets to apply term hierarchichaly. Currently only containers and datasets are supported",
+    )
+    term_urn: str = Field(default="")
+
+
+class OwnerApplyConfig(ConfigModel):
+    assets: List[str] = Field(
+        default_factory=list,
+        description="List of assets to apply owner hierarchichaly. Currently only containers and datasets are supported",
+    )
+    owner_urn: str = Field(default="")
+
+
+class DataHubApplyConfig(ConfigModel):
+    domain_apply: Optional[List[DomainApplyConfig]] = Field(
+        default=None,
+        description="List to apply domains to assets",
+    )
+    tag_apply: Optional[List[TagApplyConfig]] = Field(
+        default=None,
+        description="List to apply tags to assets",
+    )
+    term_apply: Optional[List[TermApplyConfig]] = Field(
+        default=None,
+        description="List to apply terms to assets",
+    )
+    owner_apply: Optional[List[OwnerApplyConfig]] = Field(
+        default=None,
+        description="List to apply owners to assets",
+    )
+
+
+@platform_name("DataHubApply")
+@config_class(DataHubApplyConfig)
+@support_status(SupportStatus.TESTING)
+class DataHubApplySource(Source):
+    """
+    This source is a helper over CLI
+    so people can use the helper to apply various metadata changes to DataHub
+    via Managed Ingestion
+    """
+
+    def __init__(self, ctx: PipelineContext, config: DataHubApplyConfig):
+        self.ctx = ctx
+        self.config = config
+        self.report = SourceReport()
+        self.graph = ctx.require_graph()
+
+    def _yield_workunits(
+        self,
+        proposals: List[
+            Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]
+        ],
+    ) -> Iterable[MetadataWorkUnit]:
+        for proposal in proposals:
+            if isinstance(proposal, MetadataChangeProposalWrapper):
+                yield proposal.as_workunit()
+            else:
+                yield MetadataWorkUnit(
+                    id=MetadataWorkUnit.generate_workunit_id(proposal),
+                    mcp_raw=proposal,
+                )
+
+    def _handle_assets(
+        self, assets: List[str], apply_urn: str, apply_type: str
+    ) -> Iterable[MetadataWorkUnit]:
+        for asset in assets:
+            change_proposals = apply_association_to_container(
+                asset, apply_urn, apply_type, emit=False, graph=self.graph
+            )
+            assert change_proposals is not None
+            yield from self._yield_workunits(change_proposals)
+
+    def _yield_domain(self) -> Iterable[MetadataWorkUnit]:
+        if not self.config.domain_apply:
+            return
+        for apply in self.config.domain_apply:
+            yield from self._handle_assets(apply.assets, apply.domain_urn, "domain")
+
+    def _yield_tag(self) -> Iterable[MetadataWorkUnit]:
+        if not self.config.tag_apply:
+            return
+        for apply in self.config.tag_apply:
+            yield from self._handle_assets(apply.assets, apply.tag_urn, "tag")
+
+    def _yield_term(self) -> Iterable[MetadataWorkUnit]:
+        if not self.config.term_apply:
+            return
+        for apply in self.config.term_apply:
+            yield from self._handle_assets(apply.assets, apply.term_urn, "term")
+
+    def _yield_owner(self) -> Iterable[MetadataWorkUnit]:
+        if not self.config.owner_apply:
+            return
+        for apply in self.config.owner_apply:
+            yield from self._handle_assets(apply.assets, apply.owner_urn, "owner")
+
+    def get_workunits_internal(
+        self,
+    ) -> Iterable[MetadataWorkUnit]:
+        yield from self._yield_domain()
+        yield from self._yield_tag()
+        yield from self._yield_term()
+        yield from self._yield_owner()
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [partial(auto_workunit_reporter, self.get_report())]
+
+    def get_report(self) -> SourceReport:
+        return self.report

datahub/ingestion/source/looker/looker_config.py

@@ -177,7 +177,9 @@ def _get_generic_definition(
 class LookerConnectionDefinition(ConfigModel):
     platform: str
     default_db: str
-    default_schema: Optional[str]  # Optional since some sources are two-level only
+    default_schema: Optional[str] = (
+        None  # Optional since some sources are two-level only
+    )
     platform_instance: Optional[str] = None
     platform_env: Optional[str] = Field(
         default=None,

datahub/ingestion/source/looker/looker_dataclasses.py

@@ -32,6 +32,12 @@ class LookerField:
     sql: Optional[str]
 
 
+@dataclass
+class LookerConstant:
+    name: str
+    value: str
+
+
 @dataclass
 class LookerModel:
     connection: str
@@ -75,6 +81,7 @@ class LookerModel:
             try:
                 parsed = load_and_preprocess_file(
                     path=included_file,
+                    reporter=reporter,
                     source_config=source_config,
                 )
                 included_explores = parsed.get("explores", [])
@@ -217,6 +224,7 @@ class LookerModel:
                 try:
                     parsed = load_and_preprocess_file(
                         path=included_file,
+                        reporter=reporter,
                         source_config=source_config,
                     )
                     seen_so_far.add(included_file)

datahub/ingestion/source/looker/looker_file_loader.py

@@ -4,7 +4,10 @@ from dataclasses import replace
 from typing import Dict, Optional
 
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
-from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile
+from datahub.ingestion.source.looker.looker_dataclasses import (
+    LookerConstant,
+    LookerViewFile,
+)
 from datahub.ingestion.source.looker.looker_template_language import (
     load_and_preprocess_file,
 )
@@ -30,12 +33,14 @@ class LookerViewFileLoader:
         base_projects_folder: Dict[str, pathlib.Path],
         reporter: LookMLSourceReport,
         source_config: LookMLSourceConfig,
+        manifest_constants: Dict[str, LookerConstant] = {},
     ) -> None:
         self.viewfile_cache: Dict[str, Optional[LookerViewFile]] = {}
         self._root_project_name = root_project_name
         self._base_projects_folder = base_projects_folder
         self.reporter = reporter
         self.source_config = source_config
+        self.manifest_constants = manifest_constants
 
     def _load_viewfile(
         self, project_name: str, path: str, reporter: LookMLSourceReport
@@ -60,7 +65,7 @@ class LookerViewFileLoader:
             with open(path) as file:
                 raw_file_content = file.read()
         except Exception as e:
-            self.reporter.failure(
+            self.reporter.report_warning(
                 title="LKML File Loading Error",
                 message="A lookml file is not present on local storage or GitHub",
                 context=f"file path: {path}",
@@ -71,9 +76,15 @@ class LookerViewFileLoader:
         try:
             logger.debug(f"Loading viewfile {path}")
 
+            # load_and preprocess_file is called multiple times for loading view file from multiple flows.
+            # Flag resolve_constants is a hack to avoid passing around manifest_constants from all of the flows.
+            # This is fine as rest of flows do not need resolution of constants.
             parsed = load_and_preprocess_file(
                 path=path,
+                reporter=self.reporter,
                 source_config=self.source_config,
+                resolve_constants=True,
+                manifest_constants=self.manifest_constants,
             )
 
             looker_viewfile = LookerViewFile.from_looker_dict(
@@ -90,7 +101,7 @@ class LookerViewFileLoader:
             self.viewfile_cache[path] = looker_viewfile
             return looker_viewfile
         except Exception as e:
-            self.reporter.failure(
+            self.reporter.report_warning(
                 title="LKML File Parsing Error",
                 message="The input file is not lookml file",
                 context=f"file path: {path}",