acryl-datahub 0.15.0.5rc4__py3-none-any.whl → 0.15.0.5rc6__py3-none-any.whl

datahub/ingestion/source/looker/looker_template_language.py

@@ -2,7 +2,7 @@ import logging
 import pathlib
 import re
 from abc import ABC, abstractmethod
-from typing import Any, ClassVar, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set, Union
 
 from deepmerge import always_merger
 from liquid import Undefined
@@ -27,8 +27,12 @@ from datahub.ingestion.source.looker.looker_liquid_tag import (
 from datahub.ingestion.source.looker.lookml_config import (
     DERIVED_VIEW_PATTERN,
     LookMLSourceConfig,
+    LookMLSourceReport,
 )
 
+if TYPE_CHECKING:
+    from datahub.ingestion.source.looker.looker_dataclasses import LookerConstant
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,7 +86,12 @@ class SpecialVariable:
         return self._create_new_liquid_variables_with_default(variables=variables)
 
 
-def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
+def resolve_liquid_variable(
+    text: str,
+    view_name: str,
+    liquid_variable: Dict[Any, Any],
+    report: LookMLSourceReport,
+) -> str:
     # Set variable value to NULL if not present in liquid_variable dictionary
     Undefined.__str__ = lambda instance: "NULL"  # type: ignore
     try:
@@ -96,6 +105,7 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
         # Resolve liquid template
         return create_template(text).render(liquid_variable)
     except LiquidSyntaxError as e:
+        # TODO: Will add warning once we get rid of duplcate warning message for same view
         logger.warning(f"Unsupported liquid template encountered. error [{e.message}]")
         # TODO: There are some tag specific to looker and python-liquid library does not understand them. currently
         #  we are not parsing such liquid template.
@@ -103,6 +113,7 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str:
         # See doc: https://cloud.google.com/looker/docs/templated-filters and look for { % condition region %}
         # order.region { % endcondition %}
     except CustomTagException as e:
+        # TODO: Will add warning once we get rid of duplcate warning message for same view
         logger.warning(e)
         logger.debug(e, exc_info=e)
 
@@ -192,15 +203,20 @@ class LookMLViewTransformer(ABC):
 
     source_config: LookMLSourceConfig
 
-    def __init__(self, source_config: LookMLSourceConfig):
+    def __init__(
+        self,
+        source_config: LookMLSourceConfig,
+        reporter: LookMLSourceReport,
+    ):
         self.source_config = source_config
+        self.reporter = reporter
 
     def transform(self, view: dict) -> dict:
         value_to_transform: Optional[str] = None
 
-        # is_attribute_supported check is required because not all transformer works on all attributes in current
-        # case mostly all transformer works on sql_table_name and derived.sql attributes,
-        # however IncompleteSqlTransformer only transform the derived.sql attribute
+        # is_attribute_supported check is required because not all transformers work on all attributes in the current
+        #  case, mostly all transformers work on sql_table_name and derived.sql attributes;
+        # however, IncompleteSqlTransformer only transform the derived.sql attribute
         if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME):
             # Give precedence to already processed transformed view.sql_table_name to apply more transformation
             value_to_transform = view.get(
@@ -252,7 +268,9 @@ class LiquidVariableTransformer(LookMLViewTransformer):
     def _apply_transformation(self, value: str, view: dict) -> str:
         return resolve_liquid_variable(
             text=value,
-            liquid_variable=self.source_config.liquid_variable,
+            liquid_variable=self.source_config.liquid_variables,
+            view_name=view["name"],
+            report=self.reporter,
         )
 
 
@@ -287,7 +305,7 @@ class IncompleteSqlTransformer(LookMLViewTransformer):
 
 class DropDerivedViewPatternTransformer(LookMLViewTransformer):
     """
-    drop ${} from datahub_transformed_sql_table_name and  view["derived_table"]["datahub_transformed_sql_table_name"] values.
+    drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values.
 
     Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME
     """
@@ -308,8 +326,8 @@ class LookMlIfCommentTransformer(LookMLViewTransformer):
     evaluate_to_true_regx: str
     remove_if_comment_line_regx: str
 
-    def __init__(self, source_config: LookMLSourceConfig):
-        super().__init__(source_config=source_config)
+    def __init__(self, source_config: LookMLSourceConfig, reporter: LookMLSourceReport):
+        super().__init__(source_config=source_config, reporter=reporter)
 
         # This regx will keep whatever after -- if looker_environment --
         self.evaluate_to_true_regx = r"-- if {} --".format(
@@ -335,6 +353,61 @@ class LookMlIfCommentTransformer(LookMLViewTransformer):
         return self._apply_regx(value)
 
 
+class LookmlConstantTransformer(LookMLViewTransformer):
+    """
+    Replace LookML constants @{constant} from the manifest/configuration.
+    """
+
+    CONSTANT_PATTERN = r"@{(\w+)}"  # Matches @{constant}
+
+    def __init__(
+        self,
+        source_config: LookMLSourceConfig,
+        reporter: LookMLSourceReport,
+        manifest_constants: Dict[str, "LookerConstant"],
+    ):
+        super().__init__(source_config=source_config, reporter=reporter)
+        self.manifest_constants = manifest_constants
+
+    def resolve_lookml_constant(self, text: str, view_name: Optional[str]) -> str:
+        """
+        Resolves LookML constants (@{ }) from manifest or config.
+        Logs warnings for misplaced or missing variables.
+        """
+
+        def replace_constants(match):
+            key = match.group(1)
+            # Resolve constant from config
+            if key in self.source_config.lookml_constants:
+                return str(self.source_config.lookml_constants.get(key))
+
+            # Resolve constant from manifest
+            if key in self.manifest_constants:
+                return self.manifest_constants[key].value
+
+            # Check if it's a misplaced lookml constant
+            if key in self.source_config.liquid_variables:
+                self.reporter.warning(
+                    title="Misplaced lookml constant",
+                    message="Use 'lookml_constants' instead of 'liquid_variables'.",
+                    context=f"Key {key}",
+                )
+                return f"@{{{key}}}"
+
+            self.reporter.warning(
+                title="LookML constant not found",
+                message="The constant is missing. Either add it under 'lookml_constants' in the config or define it in `manifest.lkml`.",
+                context=f"view-name: {view_name}, constant: {key}",
+            )
+            return f"@{{{key}}}"
+
+        # Resolve @{} (constant)
+        return re.sub(self.CONSTANT_PATTERN, replace_constants, text)
+
+    def _apply_transformation(self, value: str, view: dict) -> str:
+        return self.resolve_lookml_constant(text=value, view_name=view.get("name"))
+
+
 class TransformedLookMlView:
     """
     TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view.
@@ -390,24 +463,35 @@ class TransformedLookMlView:
 def process_lookml_template_language(
     source_config: LookMLSourceConfig,
     view_lkml_file_dict: dict,
+    reporter: LookMLSourceReport,
+    manifest_constants: Dict[str, "LookerConstant"] = {},
+    resolve_constants: bool = False,
 ) -> None:
     if "views" not in view_lkml_file_dict:
         return
 
     transformers: List[LookMLViewTransformer] = [
         LookMlIfCommentTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # First evaluate the -- if -- comments. Looker does the same
         LiquidVariableTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # Now resolve liquid variables
         DropDerivedViewPatternTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # Remove any ${} symbol
         IncompleteSqlTransformer(
-            source_config=source_config
+            source_config=source_config, reporter=reporter
         ),  # complete any incomplete sql
     ]
+    if resolve_constants:
+        transformers.append(
+            LookmlConstantTransformer(
+                source_config=source_config,
+                manifest_constants=manifest_constants,
+                reporter=reporter,
+            ),  # Resolve @{} constant with its corresponding value
+        )
 
     transformed_views: List[dict] = []
 
@@ -422,12 +506,18 @@ def process_lookml_template_language(
 def load_and_preprocess_file(
     path: Union[str, pathlib.Path],
     source_config: LookMLSourceConfig,
+    reporter: LookMLSourceReport,
+    manifest_constants: Dict[str, "LookerConstant"] = {},
+    resolve_constants: bool = False,
 ) -> dict:
     parsed = load_lkml(path)
 
     process_lookml_template_language(
         view_lkml_file_dict=parsed,
+        reporter=reporter,
         source_config=source_config,
+        manifest_constants=manifest_constants,
+        resolve_constants=resolve_constants,
     )
 
     return parsed

datahub/ingestion/source/looker/lookml_config.py

@@ -161,13 +161,27 @@ class LookMLSourceConfig(
         description="When enabled, looker refinement will be processed to adapt an existing view.",
     )
 
-    liquid_variable: Dict[Any, Any] = Field(
+    liquid_variables: Dict[Any, Any] = Field(
         {},
-        description="A dictionary containing Liquid variables and their corresponding values, utilized in SQL-defined "
+        description="A dictionary containing Liquid variables with their corresponding values, utilized in SQL-defined "
         "derived views. The Liquid template will be resolved in view.derived_table.sql and "
         "view.sql_table_name. Defaults to an empty dictionary.",
     )
 
+    _liquid_variable_deprecated = pydantic_renamed_field(
+        old_name="liquid_variable", new_name="liquid_variables", print_warning=True
+    )
+
+    lookml_constants: Dict[str, str] = Field(
+        {},
+        description=(
+            "A dictionary containing LookML constants (`@{constant_name}`) and their values. "
+            "If a constant is defined in the `manifest.lkml` file, its value will be used. "
+            "If not found in the manifest, the value from this config will be used instead. "
+            "Defaults to an empty dictionary."
+        ),
+    )
+
     looker_environment: Literal["prod", "dev"] = Field(
         "prod",
         description="A looker prod or dev environment. "

datahub/ingestion/source/looker/lookml_source.py

@@ -43,6 +43,7 @@ from datahub.ingestion.source.looker.looker_common import (
 from datahub.ingestion.source.looker.looker_connection import (
     get_connection_def_based_on_connection_string,
 )
+from datahub.ingestion.source.looker.looker_dataclasses import LookerConstant
 from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI
 from datahub.ingestion.source.looker.looker_template_language import (
     load_and_preprocess_file,
@@ -254,6 +255,7 @@ class LookerManifest:
     # This must be set if the manifest has local_dependency entries.
     # See https://cloud.google.com/looker/docs/reference/param-manifest-project-name
     project_name: Optional[str]
+    constants: Optional[List[Dict[str, str]]]
 
     local_dependencies: List[str]
     remote_dependencies: List[LookerRemoteDependency]
@@ -310,11 +312,14 @@ class LookMLSource(StatefulIngestionSourceBase):
                     "manage_models permission enabled on this API key."
                 ) from err
 
+        self.manifest_constants: Dict[str, "LookerConstant"] = {}
+
     def _load_model(self, path: str) -> LookerModel:
         logger.debug(f"Loading model from file {path}")
 
         parsed = load_and_preprocess_file(
             path=path,
+            reporter=self.reporter,
             source_config=self.source_config,
         )
 
@@ -500,27 +505,33 @@ class LookMLSource(StatefulIngestionSourceBase):
 
     def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManifest]:
         manifest_file = folder / "manifest.lkml"
-        if manifest_file.exists():
-            manifest_dict = load_and_preprocess_file(
-                path=manifest_file, source_config=self.source_config
-            )
 
-            manifest = LookerManifest(
-                project_name=manifest_dict.get("project_name"),
-                local_dependencies=[
-                    x["project"] for x in manifest_dict.get("local_dependencys", [])
-                ],
-                remote_dependencies=[
-                    LookerRemoteDependency(
-                        name=x["name"], url=x["url"], ref=x.get("ref")
-                    )
-                    for x in manifest_dict.get("remote_dependencys", [])
-                ],
+        if not manifest_file.exists():
+            self.reporter.info(
+                message="manifest.lkml file missing from project",
+                context=str(manifest_file),
             )
-            return manifest
-        else:
             return None
 
+        manifest_dict = load_and_preprocess_file(
+            path=manifest_file,
+            source_config=self.source_config,
+            reporter=self.reporter,
+        )
+
+        manifest = LookerManifest(
+            project_name=manifest_dict.get("project_name"),
+            constants=manifest_dict.get("constants", []),
+            local_dependencies=[
+                x["project"] for x in manifest_dict.get("local_dependencys", [])
+            ],
+            remote_dependencies=[
+                LookerRemoteDependency(name=x["name"], url=x["url"], ref=x.get("ref"))
+                for x in manifest_dict.get("remote_dependencys", [])
+            ],
+        )
+        return manifest
+
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
@@ -575,7 +586,10 @@ class LookMLSource(StatefulIngestionSourceBase):
                 self.base_projects_folder[project] = p_ref
 
             self._recursively_check_manifests(
-                tmp_dir, BASE_PROJECT_NAME, visited_projects
+                tmp_dir,
+                BASE_PROJECT_NAME,
+                visited_projects,
+                self.manifest_constants,
             )
 
             yield from self.get_internal_workunits()
@@ -588,7 +602,11 @@ class LookMLSource(StatefulIngestionSourceBase):
                 )
 
     def _recursively_check_manifests(
-        self, tmp_dir: str, project_name: str, project_visited: Set[str]
+        self,
+        tmp_dir: str,
+        project_name: str,
+        project_visited: Set[str],
+        manifest_constants: Dict[str, "LookerConstant"],
     ) -> None:
         if project_name in project_visited:
             return
@@ -605,6 +623,14 @@ class LookMLSource(StatefulIngestionSourceBase):
         if not manifest:
             return
 
+        if manifest.constants:
+            for constant in manifest.constants:
+                if constant.get("name") and constant.get("value"):
+                    manifest_constants[constant["name"]] = LookerConstant(
+                        name=constant["name"],
+                        value=constant["value"],
+                    )
+
         # Special case handling if the root project has a name in the manifest file.
         if project_name == BASE_PROJECT_NAME and manifest.project_name:
             if (
@@ -664,21 +690,27 @@ class LookMLSource(StatefulIngestionSourceBase):
                 project_visited.add(project_name)
             else:
                 self._recursively_check_manifests(
-                    tmp_dir, remote_project.name, project_visited
+                    tmp_dir,
+                    remote_project.name,
+                    project_visited,
+                    manifest_constants,
                 )
 
         for project in manifest.local_dependencies:
-            self._recursively_check_manifests(tmp_dir, project, project_visited)
+            self._recursively_check_manifests(
+                tmp_dir, project, project_visited, manifest_constants
+            )
 
     def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
         assert self.source_config.base_folder
-
         viewfile_loader = LookerViewFileLoader(
             self.source_config.project_name,
             self.base_projects_folder,
             self.reporter,
             self.source_config,
+            self.manifest_constants,
         )
+        logger.debug(f"LookML Constants : {', '.join(self.manifest_constants.keys())}")
 
         # Some views can be mentioned by multiple 'include' statements and can be included via different connections.
 

datahub/ingestion/source/snowflake/snowflake_v2.py

@@ -5,6 +5,7 @@ import logging
 import os
 import os.path
 import platform
+import re
 from dataclasses import dataclass
 from typing import Dict, Iterable, List, Optional, Union
 
@@ -33,6 +34,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.snowflake.constants import (
     GENERIC_PERMISSION_ERROR_KEY,
     SnowflakeEdition,
+    SnowflakeObjectDomain,
 )
 from datahub.ingestion.source.snowflake.snowflake_assertion import (
     SnowflakeAssertionsHandler,
@@ -162,6 +164,8 @@ class SnowflakeV2Source(
         self.data_dictionary = SnowflakeDataDictionary(connection=self.connection)
         self.lineage_extractor: Optional[SnowflakeLineageExtractor] = None
 
+        self.discovered_datasets: Optional[List[str]] = None
+
         self.aggregator: SqlParsingAggregator = self._exit_stack.enter_context(
             SqlParsingAggregator(
                 platform=self.identifiers.platform,
@@ -182,6 +186,8 @@ class SnowflakeV2Source(
                 generate_usage_statistics=False,
                 generate_operations=False,
                 format_queries=self.config.format_sql_queries,
+                is_temp_table=self._is_temp_table,
+                is_allowed_table=self._is_allowed_table,
             )
         )
         self.report.sql_aggregator = self.aggregator.report
@@ -444,6 +450,34 @@ class SnowflakeV2Source(
 
         return _report
 
+    def _is_temp_table(self, name: str) -> bool:
+        if any(
+            re.match(pattern, name, flags=re.IGNORECASE)
+            for pattern in self.config.temporary_tables_pattern
+        ):
+            return True
+
+        # This is also a temp table if
+        #   1. this name would be allowed by the dataset patterns, and
+        #   2. we have a list of discovered tables, and
+        #   3. it's not in the discovered tables list
+        if (
+            self.filters.is_dataset_pattern_allowed(name, SnowflakeObjectDomain.TABLE)
+            and self.discovered_datasets
+            and name not in self.discovered_datasets
+        ):
+            return True
+
+        return False
+
+    def _is_allowed_table(self, name: str) -> bool:
+        if self.discovered_datasets and name not in self.discovered_datasets:
+            return False
+
+        return self.filters.is_dataset_pattern_allowed(
+            name, SnowflakeObjectDomain.TABLE
+        )
+
     def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
         return [
             *super().get_workunit_processors(),
@@ -513,7 +547,7 @@ class SnowflakeV2Source(
             )
             return
 
-        discovered_datasets = discovered_tables + discovered_views
+        self.discovered_datasets = discovered_tables + discovered_views
 
         if self.config.use_queries_v2:
             with self.report.new_stage(f"*: {VIEW_PARSING}"):
@@ -538,13 +572,14 @@ class SnowflakeV2Source(
                     filters=self.filters,
                     identifiers=self.identifiers,
                     schema_resolver=schema_resolver,
-                    discovered_tables=discovered_datasets,
+                    discovered_tables=self.discovered_datasets,
                     graph=self.ctx.graph,
                 )
 
                 # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs
                 # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors,
                 # it should be pretty straightforward to refactor this and only initialize the aggregator once.
+                # This also applies for the _is_temp_table and _is_allowed_table methods above, duplicated from SnowflakeQueriesExtractor.
                 self.report.queries_extractor = queries_extractor.report
                 yield from queries_extractor.get_workunits_internal()
                 queries_extractor.close()
@@ -568,12 +603,14 @@ class SnowflakeV2Source(
             if (
                 self.config.include_usage_stats or self.config.include_operational_stats
             ) and self.usage_extractor:
-                yield from self.usage_extractor.get_usage_workunits(discovered_datasets)
+                yield from self.usage_extractor.get_usage_workunits(
+                    self.discovered_datasets
+                )
 
         if self.config.include_assertion_results:
             yield from SnowflakeAssertionsHandler(
                 self.config, self.report, self.connection, self.identifiers
-            ).get_assertion_workunits(discovered_datasets)
+            ).get_assertion_workunits(self.discovered_datasets)
 
         self.connection.close()