acryl-datahub 0.15.0.5rc3__py3-none-any.whl → 0.15.0.5rc5__py3-none-any.whl

{acryl_datahub-0.15.0.5rc3.dist-info → acryl_datahub-0.15.0.5rc5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=y-G1yv1bSuxJLKtiCJEKjyAqf2KI3SupqeNUfVB1MUg,324
+datahub/_version.py,sha256=k2KgFitDFziMKt2iil2MW_OYVP7lggtvH9A6OAJFD9c,324
 datahub/entrypoints.py,sha256=osv2ailvuW-HHlAE0fOtyblJI1X7HInZutd9DC66jqQ,8022
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/_codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -219,7 +219,7 @@ datahub/ingestion/source/abs/report.py,sha256=fzkTdTewYlWrTk4f2Cyl-e8RV4qw9wEVtm
 datahub/ingestion/source/abs/source.py,sha256=cuMezUzr-Smp5tok2ceYor5I5jp52NDMjfeN8kfIbvg,24816
 datahub/ingestion/source/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/aws/aws_common.py,sha256=DfdQgkJ_s2isFx8WvqKTlAcBk4KE8SgfpmA5BgC3fgY,17716
-datahub/ingestion/source/aws/glue.py,sha256=qwkZMcbBlHIdhhuRj-gHNYMeuMADrvaHcN3gik0n_08,57919
+datahub/ingestion/source/aws/glue.py,sha256=DwROr923M01QnvImUbMoHS6TTTT9kBz2tEmQ3Sv4EoY,58019
 datahub/ingestion/source/aws/s3_boto_utils.py,sha256=Y54jlLV5gLcuZ4Zs57kIW5dYHD89RSFfsVNlFbRnSkQ,3901
 datahub/ingestion/source/aws/s3_util.py,sha256=OFypcgmVC6jnZM90-gjcPpAMtTV1lbnreCaMhCzNlzs,2149
 datahub/ingestion/source/aws/sagemaker.py,sha256=Bl2tkBYnrindgx61VHYgNovUF_Kp_fXNcivQn28vC2w,5254
@@ -333,22 +333,22 @@ datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=-rFNXKD8_EFoX
 datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
 datahub/ingestion/source/looker/looker_common.py,sha256=squUUBHxsLeT5xbZOTO66irtOB8fL0V4Q8Tgd9EJMYU,62067
-datahub/ingestion/source/looker/looker_config.py,sha256=Kp-IoGcjfk5yPoqXFbsjo6aLA1j0H9LFEzZwhoGNvy4,13584
+datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
 datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
 datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
-datahub/ingestion/source/looker/looker_dataclasses.py,sha256=LjrP5m_A4HV-XeFlSNGVYNuyF0ulxp_qwB82Ss4Iycs,12200
-datahub/ingestion/source/looker/looker_file_loader.py,sha256=c1ewDrIb9VJg1o-asbwX9gL83kgL01vIETzzbmZIhmw,4267
+datahub/ingestion/source/looker/looker_dataclasses.py,sha256=MrDeZ4Nd0wQnJbCoI1qePYlYeObnUw5dvpWcmhKuNgc,12346
+datahub/ingestion/source/looker/looker_file_loader.py,sha256=PEyL9KWRaFcrvOkapU8wSNlFbmetmBy9tAyCgeVDOa4,4864
 datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=0gaYjBv4wkbbLWVgvaAV6JyWAFb0utTG6TCve2d9xss,11511
 datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=mO4G4MNA4YZFvZaDBpdiJ2vP3irC82kY34RdaK4Pbfs,3100
 datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
 datahub/ingestion/source/looker/looker_source.py,sha256=S-g06Bm3sbyD0Qjra9hEhZmsVDb-BY_-bCPDwCjtEoQ,66427
-datahub/ingestion/source/looker/looker_template_language.py,sha256=mfbU27NYs0mkZHXdtvS38FC5WCJ4S_aGjC8t09yecKY,14330
+datahub/ingestion/source/looker/looker_template_language.py,sha256=W-SMICKBfIuivrHywHRYchz9SJiXhoU8VOEKGQW_1v8,17825
 datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
 datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
 datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
-datahub/ingestion/source/looker/lookml_config.py,sha256=Q0fMsu_Cvm8807R6VB14VJDLqjoLTyGF-WsiUD6xEk8,10519
+datahub/ingestion/source/looker/lookml_config.py,sha256=RuZkH3DDmII21gEsUvPsJi5gxWngbYkqBP06H8_n_Hs,11353
 datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
-datahub/ingestion/source/looker/lookml_source.py,sha256=jp58gSrWXITwxd-C5UfVoLJXpxBe5smFjdJyYza-Aek,40436
+datahub/ingestion/source/looker/lookml_source.py,sha256=PJBUJgZfZyvmasDf_LJC39SggLCA6vSfAbf1PdzviZU,43889
 datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
 datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
 datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
@@ -446,7 +446,7 @@ datahub/ingestion/source/snowflake/snowflake_summary.py,sha256=kTmuCtRnvHqM8WBYh
 datahub/ingestion/source/snowflake/snowflake_tag.py,sha256=TN_cTF4a8V_tbeR2czm_hoMYfQMuqmBbFlAqyh7PJzQ,6551
 datahub/ingestion/source/snowflake/snowflake_usage_v2.py,sha256=ySFm7WDk8FW9KjCnX4HQfTqObIrlUS-V8WIHl3j0CTI,24848
 datahub/ingestion/source/snowflake/snowflake_utils.py,sha256=xq58c47zmaQPkTVqjKW25iViX8VJuHdQDTFY4jxzZ2o,12778
-datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=8nnQ_XlbT4q6y6_JleSV67njUaPjF9x1yehKu5I1lgc,32072
+datahub/ingestion/source/snowflake/snowflake_v2.py,sha256=vyTqC_C5Bf0AMRVyoxUfl1CdlgeQouX20msP2FsMqnk,33439
 datahub/ingestion/source/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/sql/athena.py,sha256=Uh9wGLOqAkcphffxOPIQNyXvjeRm74XIpaLb4rjqMjM,24045
 datahub/ingestion/source/sql/clickhouse.py,sha256=uSRy-HKAiGFTHVLoVtGoh23X0O1lwyYUaK8BaWkYhps,25555
@@ -993,9 +993,9 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-0.15.0.5rc3.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
-acryl_datahub-0.15.0.5rc3.dist-info/METADATA,sha256=iA_FeNAFTGBRzDr-rvVZXYUJtYerY57GdmWkuX4XCIg,173382
-acryl_datahub-0.15.0.5rc3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-acryl_datahub-0.15.0.5rc3.dist-info/entry_points.txt,sha256=xnPSPLK3bJGADxe4TDS4wL4u0FT_PGlahDa-ENYdYCQ,9512
-acryl_datahub-0.15.0.5rc3.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-0.15.0.5rc3.dist-info/RECORD,,
+acryl_datahub-0.15.0.5rc5.dist-info/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-0.15.0.5rc5.dist-info/METADATA,sha256=iUCOkI7iz8GUQSvFsn9nLdzi1GxoElLbxrV96MnC9BM,173382
+acryl_datahub-0.15.0.5rc5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+acryl_datahub-0.15.0.5rc5.dist-info/entry_points.txt,sha256=xnPSPLK3bJGADxe4TDS4wL4u0FT_PGlahDa-ENYdYCQ,9512
+acryl_datahub-0.15.0.5rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-0.15.0.5rc5.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "0.15.0.5rc3"
+__version__ = "0.15.0.5rc5"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/source/aws/glue.py

@@ -218,6 +218,7 @@ class GlueSourceConfig(
 
 @dataclass
 class GlueSourceReport(StaleEntityRemovalSourceReport):
+    catalog_id: Optional[str] = None
     tables_scanned = 0
     filtered: List[str] = dataclass_field(default_factory=list)
     databases: EntityFilterReport = EntityFilterReport.field(type="database")
@@ -315,6 +316,7 @@ class GlueSource(StatefulIngestionSourceBase):
         self.extract_owners = config.extract_owners
         self.source_config = config
         self.report = GlueSourceReport()
+        self.report.catalog_id = self.source_config.catalog_id
         self.glue_client = config.glue_client
         self.s3_client = config.s3_client
         self.extract_transforms = config.extract_transforms

datahub/ingestion/source/looker/looker_config.py

@@ -177,7 +177,9 @@ def _get_generic_definition(
 class LookerConnectionDefinition(ConfigModel):
     platform: str
     default_db: str
-    default_schema: Optional[str]  # Optional since some sources are two-level only
+    default_schema: Optional[str] = (
+        None  # Optional since some sources are two-level only
+    )
     platform_instance: Optional[str] = None
     platform_env: Optional[str] = Field(
         default=None,

datahub/ingestion/source/looker/looker_dataclasses.py

@@ -32,6 +32,12 @@ class LookerField:
     sql: Optional[str]
 
 
+@dataclass
+class LookerConstant:
+    name: str
+    value: str
+
+
 @dataclass
 class LookerModel:
     connection: str
@@ -75,6 +81,7 @@ class LookerModel:
             try:
                 parsed = load_and_preprocess_file(
                     path=included_file,
+                    reporter=reporter,
                     source_config=source_config,
                 )
                 included_explores = parsed.get("explores", [])
@@ -217,6 +224,7 @@ class LookerModel:
                 try:
                     parsed = load_and_preprocess_file(
                         path=included_file,
+                        reporter=reporter,
                         source_config=source_config,
                     )
                     seen_so_far.add(included_file)

datahub/ingestion/source/looker/looker_file_loader.py

@@ -4,7 +4,10 @@ from dataclasses import replace
 from typing import Dict, Optional
 
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
-from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile
+from datahub.ingestion.source.looker.looker_dataclasses import (
+    LookerConstant,
+    LookerViewFile,
+)
 from datahub.ingestion.source.looker.looker_template_language import (
     load_and_preprocess_file,
 )
@@ -30,12 +33,14 @@ class LookerViewFileLoader:
         base_projects_folder: Dict[str, pathlib.Path],
         reporter: LookMLSourceReport,
         source_config: LookMLSourceConfig,
+        manifest_constants: Dict[str, LookerConstant] = {},
     ) -> None:
         self.viewfile_cache: Dict[str, Optional[LookerViewFile]] = {}
         self._root_project_name = root_project_name
         self._base_projects_folder = base_projects_folder
         self.reporter = reporter
         self.source_config = source_config
+        self.manifest_constants = manifest_constants
 
     def _load_viewfile(
         self, project_name: str, path: str, reporter: LookMLSourceReport
@@ -60,7 +65,7 @@ class LookerViewFileLoader:
             with open(path) as file:
                 raw_file_content = file.read()
         except Exception as e:
-            self.reporter.failure(
+            self.reporter.report_warning(
                 title="LKML File Loading Error",
                 message="A lookml file is not present on local storage or GitHub",
                 context=f"file path: {path}",
@@ -71,9 +76,15 @@ class LookerViewFileLoader:
         try:
             logger.debug(f"Loading viewfile {path}")
 
+            # load_and preprocess_file is called multiple times for loading view file from multiple flows.
+            # Flag resolve_constants is a hack to avoid passing around manifest_constants from all of the flows.
+            # This is fine as rest of flows do not need resolution of constants.
             parsed = load_and_preprocess_file(
                 path=path,
+                reporter=self.reporter,
                 source_config=self.source_config,
+                resolve_constants=True,
+                manifest_constants=self.manifest_constants,
             )
 
             looker_viewfile = LookerViewFile.from_looker_dict(
@@ -90,7 +101,7 @@ class LookerViewFileLoader:
             self.viewfile_cache[path] = looker_viewfile
             return looker_viewfile
         except Exception as e:
-            self.reporter.failure(
+            self.reporter.report_warning(
                 title="LKML File Parsing Error",
                 message="The input file is not lookml file",
                 context=f"file path: {path}",

datahub/ingestion/source/looker/looker_template_language.py

@@ -2,7 +2,7 @@ import logging
 import pathlib
 import re
 from abc import ABC, abstractmethod
-from typing import Any, ClassVar, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set, Union
 
 from deepmerge import always_merger
 from liquid import Undefined
@@ -27,8 +27,12 @@ from datahub.ingestion.source.looker.looker_liquid_tag import (
 from datahub.ingestion.source.looker.lookml_config import (
     DERIVED_VIEW_PATTERN,
     LookMLSourceConfig,
+    LookMLSourceReport,
 )
 
+if TYPE_CHECKING:
+    from datahub.ingestion.source.looker.looker_dataclasses import LookerConstant
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,7 +86,12 @@ class SpecialVariable:
         return self._create_new_liquid_variables_with_default(variables=variables)
 
 
-def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
+def resolve_liquid_variable(
+    text: str,
+    view_name: str,
+    liquid_variable: Dict[Any, Any],
+    report: LookMLSourceReport,
+) -> str:
     # Set variable value to NULL if not present in liquid_variable dictionary
     Undefined.__str__ = lambda instance: "NULL"  # type: ignore
     try:
@@ -96,6 +105,7 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
         # Resolve liquid template
         return create_template(text).render(liquid_variable)
     except LiquidSyntaxError as e:
+        # TODO: Will add warning once we get rid of duplcate warning message for same view
         logger.warning(f"Unsupported liquid template encountered. error [{e.message}]")
         # TODO: There are some tag specific to looker and python-liquid library does not understand them. currently
         #  we are not parsing such liquid template.
@@ -103,6 +113,7 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
         # See doc: https://cloud.google.com/looker/docs/templated-filters and look for { % condition region %}
         # order.region { % endcondition %}
     except CustomTagException as e:
+        # TODO: Will add warning once we get rid of duplcate warning message for same view
         logger.warning(e)
         logger.debug(e, exc_info=e)
 
@@ -192,15 +203,20 @@ class LookMLViewTransformer(ABC):
 
     source_config: LookMLSourceConfig
 
-    def __init__(self, source_config: LookMLSourceConfig):
+    def __init__(
+        self,
+        source_config: LookMLSourceConfig,
+        reporter: LookMLSourceReport,
+    ):
         self.source_config = source_config
+        self.reporter = reporter
 
     def transform(self, view: dict) -> dict:
         value_to_transform: Optional[str] = None
 
-        # is_attribute_supported check is required because not all transformer works on all attributes in current
-        # case mostly all transformer works on sql_table_name and derived.sql attributes,
-        # however IncompleteSqlTransformer only transform the derived.sql attribute
+        # is_attribute_supported check is required because not all transformers work on all attributes in the current
+        #  case, mostly all transformers work on sql_table_name and derived.sql attributes;
+        # however, IncompleteSqlTransformer only transform the derived.sql attribute
         if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME):
             # Give precedence to already processed transformed view.sql_table_name to apply more transformation
             value_to_transform = view.get(
@@ -252,7 +268,9 @@ class LiquidVariableTransformer(LookMLViewTransformer):
     def _apply_transformation(self, value: str, view: dict) -> str:
         return resolve_liquid_variable(
             text=value,
-            liquid_variable=self.source_config.liquid_variable,
+            liquid_variable=self.source_config.liquid_variables,
+            view_name=view["name"],
+            report=self.reporter,
         )
 
 
@@ -287,7 +305,7 @@ class IncompleteSqlTransformer(LookMLViewTransformer):
 
 class DropDerivedViewPatternTransformer(LookMLViewTransformer):
     """
-    drop ${} from datahub_transformed_sql_table_name and  view["derived_table"]["datahub_transformed_sql_table_name"] values.
+    drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values.
 
     Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME
     """
@@ -308,8 +326,8 @@ class LookMlIfCommentTransformer(LookMLViewTransformer):
     evaluate_to_true_regx: str
     remove_if_comment_line_regx: str
 
-    def __init__(self, source_config: LookMLSourceConfig):
-        super().__init__(source_config=source_config)
+    def __init__(self, source_config: LookMLSourceConfig, reporter: LookMLSourceReport):
+        super().__init__(source_config=source_config, reporter=reporter)
 
         # This regx will keep whatever after -- if looker_environment --
         self.evaluate_to_true_regx = r"-- if {} --".format(
@@ -335,6 +353,61 @@ class LookMlIfCommentTransformer(LookMLViewTransformer):
         return self._apply_regx(value)
 
 
+class LookmlConstantTransformer(LookMLViewTransformer):
+    """
+    Replace LookML constants @{constant} from the manifest/configuration.
+    """
+
+    CONSTANT_PATTERN = r"@{(\w+)}"  # Matches @{constant}
+
+    def __init__(
+        self,
+        source_config: LookMLSourceConfig,
+        reporter: LookMLSourceReport,
+        manifest_constants: Dict[str, "LookerConstant"],
+    ):
+        super().__init__(source_config=source_config, reporter=reporter)
+        self.manifest_constants = manifest_constants
+
+    def resolve_lookml_constant(self, text: str, view_name: Optional[str]) -> str:
+        """
+        Resolves LookML constants (@{ }) from manifest or config.
+        Logs warnings for misplaced or missing variables.
+        """
+
+        def replace_constants(match):
+            key = match.group(1)
+            # Resolve constant from config
+            if key in self.source_config.lookml_constants:
+                return str(self.source_config.lookml_constants.get(key))
+
+            # Resolve constant from manifest
+            if key in self.manifest_constants:
+                return self.manifest_constants[key].value
+
+            # Check if it's a misplaced lookml constant
+            if key in self.source_config.liquid_variables:
+                self.reporter.warning(
+                    title="Misplaced lookml constant",
+                    message="Use 'lookml_constants' instead of 'liquid_variables'.",
+                    context=f"Key {key}",
+                )
+                return f"@{{{key}}}"
+
+            self.reporter.warning(
+                title="LookML constant not found",
+                message="The constant is missing. Either add it under 'lookml_constants' in the config or define it in `manifest.lkml`.",
+                context=f"view-name: {view_name}, constant: {key}",
+            )
+            return f"@{{{key}}}"
+
+        # Resolve @{} (constant)
+        return re.sub(self.CONSTANT_PATTERN, replace_constants, text)
+
+    def _apply_transformation(self, value: str, view: dict) -> str:
+        return self.resolve_lookml_constant(text=value, view_name=view.get("name"))
+
+
 class TransformedLookMlView:
     """
     TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view.
@@ -390,24 +463,35 @@ class TransformedLookMlView:
 def process_lookml_template_language(
     source_config: LookMLSourceConfig,
     view_lkml_file_dict: dict,
+    reporter: LookMLSourceReport,
+    manifest_constants: Dict[str, "LookerConstant"] = {},
+    resolve_constants: bool = False,
 ) -> None:
     if "views" not in view_lkml_file_dict:
         return
 
     transformers: List[LookMLViewTransformer] = [
         LookMlIfCommentTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # First evaluate the -- if -- comments. Looker does the same
         LiquidVariableTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # Now resolve liquid variables
         DropDerivedViewPatternTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # Remove any ${} symbol
         IncompleteSqlTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # complete any incomplete sql
     ]
+    if resolve_constants:
+        transformers.append(
+            LookmlConstantTransformer(
+                source_config=source_config,
+                manifest_constants=manifest_constants,
+                reporter=reporter,
+            ),  # Resolve @{} constant with its corresponding value
+        )
 
     transformed_views: List[dict] = []
 
@@ -422,12 +506,18 @@ def process_lookml_template_language(
 def load_and_preprocess_file(
     path: Union[str, pathlib.Path],
     source_config: LookMLSourceConfig,
+    reporter: LookMLSourceReport,
+    manifest_constants: Dict[str, "LookerConstant"] = {},
+    resolve_constants: bool = False,
 ) -> dict:
     parsed = load_lkml(path)
 
     process_lookml_template_language(
         view_lkml_file_dict=parsed,
+        reporter=reporter,
         source_config=source_config,
+        manifest_constants=manifest_constants,
+        resolve_constants=resolve_constants,
     )
 
     return parsed

datahub/ingestion/source/looker/lookml_config.py

@@ -139,7 +139,10 @@ class LookMLSourceConfig(
     )
     emit_reachable_views_only: bool = Field(
         True,
-        description="When enabled, only views that are reachable from explores defined in the model files are emitted",
+        description=(
+            "When enabled, only views that are reachable from explores defined in the model files are emitted. "
+            "If set to False, all views imported in model files are emitted. Views that are unreachable i.e. not explicitly defined in the model files are currently not emitted however reported as warning for debugging purposes."
+        ),
     )
     populate_sql_logic_for_missing_descriptions: bool = Field(
         False,
@@ -158,13 +161,27 @@ class LookMLSourceConfig(
         description="When enabled, looker refinement will be processed to adapt an existing view.",
     )
 
-    liquid_variable: Dict[Any, Any] = Field(
+    liquid_variables: Dict[Any, Any] = Field(
         {},
-        description="A dictionary containing Liquid variables and their corresponding values, utilized in SQL-defined "
+        description="A dictionary containing Liquid variables with their corresponding values, utilized in SQL-defined "
         "derived views. The Liquid template will be resolved in view.derived_table.sql and "
         "view.sql_table_name. Defaults to an empty dictionary.",
     )
 
+    _liquid_variable_deprecated = pydantic_renamed_field(
+        old_name="liquid_variable", new_name="liquid_variables", print_warning=True
+    )
+
+    lookml_constants: Dict[str, str] = Field(
+        {},
+        description=(
+            "A dictionary containing LookML constants (`@{constant_name}`) and their values. "
+            "If a constant is defined in the `manifest.lkml` file, its value will be used. "
+            "If not found in the manifest, the value from this config will be used instead. "
+            "Defaults to an empty dictionary."
+        ),
+    )
+
     looker_environment: Literal["prod", "dev"] = Field(
         "prod",
         description="A looker prod or dev environment. "