openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,13 +21,27 @@ Notes:
 """
 import copy
 import os
+import re
 import traceback
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Sequence, Set, Type, Union, cast
+from typing import (
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    cast,
+    get_args,
+)
 
 import giturlparse
 import lkml
+import networkx as nx
 from liquid import Template
 from looker_sdk.sdk.api40.methods import Looker40SDK
 from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard
@@ -53,8 +67,9 @@ from metadata.generated.schema.entity.data.dashboardDataModel import (
     DashboardDataModel,
     DataModelType,
 )
-from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.data.table import Column, Table
 from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
+    LocalRepositoryPath,
     LookerConnection,
     NoGitCredentials,
 )
@@ -84,15 +99,21 @@ from metadata.generated.schema.type.basic import (
     SourceUrl,
     Uuid,
 )
-from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
+from metadata.generated.schema.type.entityLineage import (
+    ColumnLineage,
+    EntitiesEdge,
+    LineageDetails,
+)
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
+from metadata.ingestion.lineage.sql_lineage import get_column_fqn
+from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import (
     DashboardServiceSource,
@@ -108,7 +129,6 @@ from metadata.ingestion.source.dashboard.looker.models import (
     LookMlView,
     ViewName,
 )
-from metadata.ingestion.source.dashboard.looker.parser import LkmlParser
 from metadata.ingestion.source.dashboard.looker.utils import _clone_repo
 from metadata.readers.file.api_reader import ReadersCredentials
 from metadata.readers.file.base import Reader
@@ -118,13 +138,17 @@ from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart, filter_by_datamodel
 from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_labels
 
 logger = ingestion_logger()
 
-
 LIST_DASHBOARD_FIELDS = ["id", "title"]
 IMPORTED_PROJECTS_DIR = "imported_projects"
 
+# we need to find the derived references in the SQL query using regex
+# https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables
+DERIVED_REFERENCES = r"\${([\w\s\d_.]+)\.SQL_TABLE_NAME}"
+
 # Here we can update the fields to get further information, such as:
 # created_at, updated_at, last_updater_id, deleted_at, deleter_id, favorite_count, last_viewed_at
 GET_DASHBOARD_FIELDS = [
@@ -141,6 +165,8 @@ GET_DASHBOARD_FIELDS = [
 TEMP_FOLDER_DIRECTORY = os.path.join(os.getcwd(), "tmp")
 REPO_TMP_LOCAL_PATH = f"{TEMP_FOLDER_DIRECTORY}/lookml_repos"
 
+LOOKER_TAG_CATEGORY = "LookerTags"
+
 
 def clean_dashboard_name(name: str) -> str:
     """
@@ -156,6 +182,13 @@ def build_datamodel_name(model_name: str, explore_name: str) -> str:
     return clean_dashboard_name(model_name + "_" + explore_name)
 
 
+def find_derived_references(sql_query: str) -> List[str]:
+    if sql_query is None:
+        return []
+    matches = re.findall(DERIVED_REFERENCES, sql_query)
+    return matches
+
+
 class LookerSource(DashboardServiceSource):
     """
     Looker Source Class.
@@ -163,6 +196,8 @@ class LookerSource(DashboardServiceSource):
     Its client uses Looker 40 from the SDK: client = looker_sdk.init40()
     """
 
+    # pylint: disable=too-many-instance-attributes, too-many-public-methods
+
     config: WorkflowSource
     metadata: OpenMetadata
     client: Looker40SDK
@@ -176,13 +211,18 @@ class LookerSource(DashboardServiceSource):
         self.today = datetime.now().strftime("%Y-%m-%d")
 
         self._explores_cache = {}
+        self._views_cache = {}
         self._repo_credentials: Optional[ReadersCredentials] = None
         self._reader_class: Optional[Type[Reader]] = None
-        self._project_parsers: Optional[Dict[str, LkmlParser]] = None
-        self._main_lookml_repo: Optional[LookMLRepo] = None
+        self._project_parsers: Optional[Dict[str, BulkLkmlParser]] = None
+        self._main_lookml_repos: Optional[List[LookMLRepo]] = None
         self._main__lookml_manifest: Optional[LookMLManifest] = None
         self._view_data_model: Optional[DashboardDataModel] = None
 
+        self._parsed_views: Optional[Dict[str, str]] = {}
+        self._unparsed_views: Optional[Dict[str, str]] = {}
+        self._derived_dependencies = nx.DiGraph()
+
         self._added_lineage: Optional[Dict] = {}
 
     @classmethod
@@ -205,62 +245,108 @@ class LookerSource(DashboardServiceSource):
         credentials: Optional[
             Union[
                 NoGitCredentials,
+                LocalRepositoryPath,
                 GitHubCredentials,
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
-        ]
-    ) -> "LookMLRepo":
-        repo_name = (
-            f"{credentials.repositoryOwner.root}/{credentials.repositoryName.root}"
-        )
-        repo_path = f"{REPO_TMP_LOCAL_PATH}/{credentials.repositoryName.root}"
-        _clone_repo(
-            repo_name,
-            repo_path,
-            credentials,
-            overwrite=True,
-        )
-        return LookMLRepo(name=repo_name, path=repo_path)
+        ],
+    ) -> List["LookMLRepo"]:
+        repos = []
+        if isinstance(credentials, LocalRepositoryPath):
+            # For local repository path, use the path directly without cloning
+            local_path = Path(credentials.root)
+            repo_name = local_path.name
+            repos.append(LookMLRepo(name=repo_name, path=str(local_path)))
+        elif isinstance(
+            credentials, (GitHubCredentials, BitBucketCredentials, GitlabCredentials)
+        ):
+            # Support comma-separated repository names
+            repository_names = [
+                name.strip()
+                for name in credentials.repositoryName.root.split(",")
+                if name.strip()
+            ]
+
+            for repo_name_only in repository_names:
+                repo_name = f"{credentials.repositoryOwner.root}/{repo_name_only}"
+                repo_path = f"{REPO_TMP_LOCAL_PATH}/{repo_name_only}"
+
+                single_repo_creds = copy.deepcopy(credentials)
+                single_repo_creds.repositoryName.root = repo_name_only
+
+                _clone_repo(
+                    repo_name,
+                    repo_path,
+                    single_repo_creds,
+                    overwrite=True,
+                )
+                repos.append(LookMLRepo(name=repo_name, path=repo_path))
+        else:
+            # For NoGitCredentials or other unsupported types
+            raise ValueError(f"Unsupported credential type: {type(credentials)}")
+
+        return repos
 
     def __read_manifest(
         self,
         credentials: Optional[
             Union[
                 NoGitCredentials,
+                LocalRepositoryPath,
                 GitHubCredentials,
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
         ],
+        repo: LookMLRepo,
         path="manifest.lkml",
     ) -> Optional[LookMLManifest]:
-        file_path = f"{self._main_lookml_repo.path}/{path}"
-        if not os.path.isfile(file_path):
+        file_path = Path(repo.path) / path
+        if not file_path.is_file():
+            if isinstance(credentials, LocalRepositoryPath):
+                logger.warning(
+                    f"Manifest file '{path}' not found in local repository at {file_path}. "
+                    f"Ensure the manifest file exists in your local LookML repository."
+                )
             return None
+
         with open(file_path, "r", encoding="utf-8") as fle:
             manifest = LookMLManifest.model_validate(lkml.load(fle))
             if manifest and manifest.remote_dependency:
                 remote_name = manifest.remote_dependency["name"]
                 remote_git_url = manifest.remote_dependency["url"]
 
-                url_parsed = giturlparse.parse(remote_git_url)
-                _clone_repo(
-                    f"{url_parsed.owner}/{url_parsed.repo}",  # pylint: disable=E1101
-                    f"{self._main_lookml_repo.path}/{IMPORTED_PROJECTS_DIR}/{remote_name}",
-                    credentials,
-                )
+                if isinstance(credentials, LocalRepositoryPath):
+                    # For local repository path, warn about remote dependencies
+                    logger.warning(
+                        f"Remote dependency '{remote_name}' found in manifest. "
+                        f"When using localRepositoryPath, remote dependencies are not automatically fetched. "
+                        f"If needed, manually place the dependency in the '{IMPORTED_PROJECTS_DIR}/{remote_name}' directory within your local repository."
+                    )
+                else:
+                    # For remote repositories, clone the dependency as before
+                    url_parsed = giturlparse.parse(remote_git_url)
+                    _clone_repo(
+                        f"{url_parsed.owner}/{url_parsed.repo}",  # pylint: disable=E1101
+                        f"{repo.path}/{IMPORTED_PROJECTS_DIR}/{remote_name}",
+                        credentials,
+                    )
 
             return manifest
 
     def prepare(self):
         if self.service_connection.gitCredentials:
             credentials = self.service_connection.gitCredentials
-            self._main_lookml_repo = self.__init_repo(credentials)
-            self._main__lookml_manifest = self.__read_manifest(credentials)
+            self._main_lookml_repos = self.__init_repo(credentials)
+            if self._main_lookml_repos:
+                # Read manifest from the first repository (primary repository)
+                self._main__lookml_manifest = self.__read_manifest(
+                    credentials, self._main_lookml_repos[0]
+                )
 
     @property
-    def parser(self) -> Optional[Dict[str, LkmlParser]]:
+    def parser(self) -> Optional[Dict[str, BulkLkmlParser]]:
         if self.repository_credentials:
             return self._project_parsers
         return None
@@ -279,15 +365,27 @@ class LookerSource(DashboardServiceSource):
         and can be accessed with the same token. If we have
         any errors obtaining the git project information, we will default
         to the incoming GitHub Credentials.
+
+        Since BulkLkmlParser uses Singleton pattern, we create ONE parser per project
+        that aggregates views from all repositories.
         """
-        if self.repository_credentials:
+        if self.repository_credentials and self._main_lookml_repos:
             all_projects: Set[str] = {model.project_name for model in all_lookml_models}
-            self._project_parsers: Dict[str, LkmlParser] = {
-                project_name: BulkLkmlParser(
-                    reader=self.reader(Path(self._main_lookml_repo.path))
+            self._project_parsers: Dict[str, BulkLkmlParser] = {}
+
+            # Create readers for all repositories
+            primary_reader = self.reader(Path(self._main_lookml_repos[0].path))
+            additional_readers = [
+                self.reader(Path(repo.path)) for repo in self._main_lookml_repos[1:]
+            ]
+
+            # For each project, create a single parser with all readers
+            for project_name in all_projects:
+                parser = BulkLkmlParser(
+                    reader=primary_reader, additional_readers=additional_readers
                 )
-                for project_name in all_projects
-            }
+                self._project_parsers[project_name] = parser
+
             logger.info(f"We found the following parsers:\n {self._project_parsers}")
 
     def get_lookml_project_credentials(self, project_name: str) -> ReadersCredentials:
@@ -324,8 +422,13 @@ class LookerSource(DashboardServiceSource):
         We either get GitHubCredentials or `NoGitHubCredentials`
         """
         if not self._repo_credentials:
-            if self.service_connection.gitCredentials and isinstance(
-                self.service_connection.gitCredentials, ReadersCredentials
+            if self.service_connection.gitCredentials and (
+                isinstance(
+                    self.service_connection.gitCredentials, get_args(ReadersCredentials)
+                )
+                or isinstance(
+                    self.service_connection.gitCredentials, LocalRepositoryPath
+                )
             ):
                 self._repo_credentials = self.service_connection.gitCredentials
 
@@ -399,6 +502,21 @@ class LookerSource(DashboardServiceSource):
         )
         return _datamodel
 
+    def yield_data_model_tags(
+        self, tags: List[str]
+    ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        Method to yield tags related to specific dashboards
+        """
+        if tags and self.source_config.includeTags:
+            yield from get_ometa_tag_and_classification(
+                tags=tags or [],
+                classification_name=LOOKER_TAG_CATEGORY,
+                tag_description="Looker Tag",
+                classification_description="Tags associated with looker entities",
+                include_tags=self.source_config.includeTags,
+            )
+
     def yield_bulk_datamodel(
         self, model: LookmlModelExplore
     ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
@@ -413,13 +531,21 @@ class LookerSource(DashboardServiceSource):
             ):
                 self.status.filter(datamodel_name, "Data model filtered out.")
             else:
+                if model.tags and self.source_config.includeTags:
+                    yield from self.yield_data_model_tags(model.tags or [])
                 explore_datamodel = CreateDashboardDataModelRequest(
                     name=EntityName(datamodel_name),
                     displayName=model.name,
-                    description=Markdown(model.description)
-                    if model.description
-                    else None,
+                    description=(
+                        Markdown(model.description) if model.description else None
+                    ),
                     service=self.context.get().dashboard_service,
+                    tags=get_tag_labels(
+                        metadata=self.metadata,
+                        tags=model.tags or [],
+                        classification_name=LOOKER_TAG_CATEGORY,
+                        include_tags=self.source_config.includeTags,
+                    ),
                     dataModelType=DataModelType.LookMlExplore.value,
                     serviceType=DashboardServiceType.Looker.value,
                     columns=get_columns_from_model(model),
@@ -443,6 +569,13 @@ class LookerSource(DashboardServiceSource):
                 # We can get VIEWs from the JOINs to know the dependencies
                 # We will only try and fetch if we have the credentials
                 if self.repository_credentials:
+                    logger.info(
+                        f"Repository credentials are present, processing views of explore model {datamodel_name}"
+                    )
+                    if model.joins:
+                        logger.info(
+                            f"Joins are present, processing views of explore model {datamodel_name}"
+                        )
                     for view in model.joins:
                         if filter_by_datamodel(
                             self.source_config.dataModelFilterPattern, view.name
@@ -456,6 +589,9 @@ class LookerSource(DashboardServiceSource):
                             view_name=ViewName(view_name), explore=model
                         )
                     if model.view_name:
+                        logger.info(
+                            f"View name is present, processing view {model.view_name} of explore model {datamodel_name}"
+                        )
                         yield from self._process_view(
                             view_name=ViewName(model.view_name), explore=model
                         )
@@ -487,9 +623,13 @@ class LookerSource(DashboardServiceSource):
             try:
                 project_parser = self.parser.get(explore.project_name)
                 if project_parser:
-                    return project_parser.parsed_files.get(
+                    explore_sql = project_parser.parsed_files.get(
                         Includes(get_path_from_link(explore.lookml_link))
                     )
+                    logger.debug(
+                        f"Explore SQL for project {explore.project_name}: \n{explore_sql}"
+                    )
+                    return explore_sql
             except Exception as err:
                 logger.warning(f"Exception getting the model sql: {err}")
 
@@ -513,15 +653,24 @@ class LookerSource(DashboardServiceSource):
             view: Optional[LookMlView] = project_parser.find_view(view_name=view_name)
 
             if view:
+                if view.tags and self.source_config.includeTags:
+                    yield from self.yield_data_model_tags(view.tags or [])
+                datamodel_view_name = (
+                    build_datamodel_name(explore.model_name, view.name) + "_view"
+                )
                 data_model_request = CreateDashboardDataModelRequest(
-                    name=EntityName(
-                        build_datamodel_name(explore.model_name, view.name)
-                    ),
+                    name=EntityName(datamodel_view_name),
                     displayName=view.name,
-                    description=Markdown(view.description)
-                    if view.description
-                    else None,
+                    description=(
+                        Markdown(view.description) if view.description else None
+                    ),
                     service=self.context.get().dashboard_service,
+                    tags=get_tag_labels(
+                        metadata=self.metadata,
+                        tags=view.tags or [],
+                        classification_name=LOOKER_TAG_CATEGORY,
+                        include_tags=self.source_config.includeTags,
+                    ),
                     dataModelType=DataModelType.LookMlView.value,
                     serviceType=DashboardServiceType.Looker.value,
                     columns=get_columns_from_model(view),
@@ -530,9 +679,8 @@ class LookerSource(DashboardServiceSource):
                     project=explore.project_name,
                 )
                 yield Either(right=data_model_request)
-                self._view_data_model = self._build_data_model(
-                    build_datamodel_name(explore.model_name, view.name)
-                )
+                self._view_data_model = self._build_data_model(datamodel_view_name)
+                self._views_cache[view.name] = self._view_data_model
                 self.register_record_datamodel(datamodel_request=data_model_request)
                 yield from self.add_view_lineage(view, explore)
             else:
@@ -544,6 +692,176 @@ class LookerSource(DashboardServiceSource):
                     )
                 )
 
+    def replace_derived_references(self, sql_query):
+        """
+        Replace all derived references with the parsed views sql query
+        will replace the derived references in the SQL query using regex
+        for e.g. It will replace ${view_name.SQL_TABLE_NAME} with the parsed view query for view_name
+        https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables
+        """
+        try:
+            sql_query = re.sub(
+                DERIVED_REFERENCES,
+                # from `${view_name.SQL_TABLE_NAME}` we want the `view_name`.
+                # match.group(1) will give us the `view_name`
+                lambda match: f"({self._parsed_views.get(match.group(1), match.group(0))})",
+                sql_query,
+            )
+        except Exception as e:
+            logger.warning(
+                f"Something went wrong while replacing derived view references: {e}"
+            )
+        return sql_query
+
+    def build_lineage_for_unparsed_views(self) -> Iterable[Either[AddLineageRequest]]:
+        """
+        build lineage by parsing the unparsed views containing derived references
+        """
+        try:
+            # Doing a reversed topological sort to process the views in the right order
+            for view_name in reversed(
+                list(nx.topological_sort(self._derived_dependencies))
+            ):
+                if view_name in self._parsed_views:
+                    # Skip if already processed
+                    continue
+                sql_query = self.replace_derived_references(
+                    self._unparsed_views[view_name]
+                )
+                if view_references := find_derived_references(sql_query):
+                    # There are still derived references in the view query
+                    logger.debug(
+                        f"Views {view_references} not found for {view_name}. Skipping."
+                    )
+                    continue
+                self._parsed_views[view_name] = sql_query
+                del self._unparsed_views[view_name]
+                yield from self._build_lineage_for_view(view_name, sql_query)
+
+        except Exception as err:
+            yield Either(
+                left=StackTraceError(
+                    name="parse_unparsed_views",
+                    error=f"Error parsing unparsed views: {err}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def _add_dependency_edge(self, view_name: str, view_references: List[str]):
+        """
+        Add a dependency edge between the view and the derived reference
+        """
+        for dependent_view_name in view_references:
+            self._derived_dependencies.add_edge(view_name, dependent_view_name)
+
+    def _extract_column_lineage(self, view: LookMlView) -> List[Tuple[Column, Column]]:
+        """
+        Extract column level lineage from a LookML view.
+        Returns a list of tuples containing (source_column, target_column)
+        """
+        logger.debug(f"Extracting column lineage for view: {view.name}")
+        column_lineage = []
+        try:
+            # Build a map: field_name → sql block
+            field_sql_map = {}
+            for field_type in ["dimensions", "measures", "dimension_groups"]:
+                for field in getattr(view, field_type, []):
+                    if hasattr(field, "sql") and field.sql is not None:
+                        field_sql_map[field.name] = field.sql
+
+            # Regex to extract ${TABLE}.col and ${field}
+            table_col_pattern = re.compile(r"\$\{TABLE\}\.([a-zA-Z_][a-zA-Z0-9_]*)")
+            dimension_ref_pattern = re.compile(
+                r"\$\{(?!TABLE\})([a-zA-Z_][a-zA-Z0-9_]*)\}"
+            )
+
+            # Recursive resolver
+            def resolve(field_name, visited=None):
+                if visited is None:
+                    visited = set()
+                if field_name in visited:
+                    return set()
+                visited.add(field_name)
+
+                sql = field_sql_map.get(field_name, "")
+                source_cols = set(table_col_pattern.findall(sql))
+                dimension_refs = dimension_ref_pattern.findall(sql)
+
+                for ref in dimension_refs:
+                    source_cols.update(resolve(ref, visited))
+
+                return source_cols
+
+            # Build lineage for each field
+            for field_name, sql in field_sql_map.items():
+                try:
+                    if not sql:  # Skip if sql is None or empty
+                        continue
+                    source_cols = resolve(field_name)
+                    for source_col in source_cols:
+                        column_lineage.append((source_col, field_name))
+                except Exception as err:
+                    logger.warning(f"Error processing field {field_name}: {err}")
+                    logger.debug(traceback.format_exc())
+                    continue
+
+            return column_lineage
+        except Exception as e:
+            logger.warning(f"Error extracting column lineage: {e}")
+            logger.debug(traceback.format_exc())
+            return []
+
+    def _get_explore_column_lineage(
+        self, explore_model: LookmlModelExplore
+    ) -> Optional[List[ColumnLineage]]:
+        """
+        Build the lineage between the view and the explore
+        """
+        processed_column_lineage = []
+        for field in explore_model.columns or []:
+            try:
+                # Look for fields with format view_name.col
+                field_name = field.name.root
+                if "." not in field_name:
+                    logger.debug(
+                        f"Field [{field_name}] does not have a view name. Skipping."
+                    )
+                    continue
+
+                view_name, col_name = field_name.split(".")
+                if view_name != self._view_data_model.displayName:
+                    logger.debug(
+                        f"View name [{view_name}] do not match the view name"
+                        f"[{self._view_data_model.displayName}] Skipping."
+                    )
+                    continue
+
+                # Add lineage from view column to explore column
+                view_col = None
+                for col in self._view_data_model.columns:
+                    if (
+                        col.displayName and col.displayName.lower() == col_name.lower()
+                    ) or (col.name.root.lower() == col_name.lower()):
+                        view_col = col
+                        break
+                from_column = view_col.fullyQualifiedName.root if view_col else None
+                to_column = self._get_data_model_column_fqn(
+                    data_model_entity=explore_model, column=str(field.name.root)
+                )
+
+                if from_column and to_column:
+                    processed_column_lineage.append(
+                        ColumnLineage(fromColumns=[from_column], toColumn=to_column)
+                    )
+            except Exception as err:
+                logger.warning(
+                    "Error processing column lineage for explore_model"
+                    f"[{explore_model.name}] field [{field.name}]: {err}"
+                )
+                logger.debug(traceback.format_exc())
+                continue
+        return processed_column_lineage
+
     def add_view_lineage(
         self, view: LookMlView, explore: LookmlModelExplore
     ) -> Iterable[Either[AddLineageRequest]]:
@@ -558,8 +876,14 @@ class LookerSource(DashboardServiceSource):
             # TODO: column-level lineage parsing the explore columns with the format `view_name.col`
             # Now the context has the newly created view
             if explore_model:
+                logger.debug(
+                    f"Building lineage request for view {self._view_data_model.name} to explore {explore_model.name}"
+                )
+                column_lineage = self._get_explore_column_lineage(explore_model)
                 yield self._get_add_lineage_request(
-                    from_entity=self._view_data_model, to_entity=explore_model
+                    from_entity=self._view_data_model,
+                    to_entity=explore_model,
+                    column_lineage=column_lineage,
                 )
 
             else:
@@ -568,43 +892,66 @@ class LookerSource(DashboardServiceSource):
                     " while processing view lineage."
                 )
 
-            db_service_names = self.get_db_service_names()
+            # Handle view-to-view lineage via extends
+            if view.extends__all:
+                for extended_views_list in view.extends__all:
+                    for extended_view_name in extended_views_list:
+                        extended_view_model = self._views_cache.get(extended_view_name)
+                        if extended_view_model:
+                            logger.debug(
+                                f"Building lineage from extended view {extended_view_name} to view {self._view_data_model.name}"
+                            )
+                            yield self._get_add_lineage_request(
+                                from_entity=extended_view_model,
+                                to_entity=self._view_data_model,
+                                column_lineage=[],
+                            )
+                        else:
+                            logger.debug(
+                                f"Extended view [{extended_view_name}] not found in cache for view [{view.name}]"
+                            )
+
+            db_service_prefixes = self.get_db_service_prefixes()
 
             if view.sql_table_name:
                 sql_table_name = self._render_table_name(view.sql_table_name)
-                source_table_name = self._clean_table_name(sql_table_name)
 
-                # View to the source is only there if we are informing the dbServiceNames
-                for db_service_name in db_service_names or []:
+                for db_service_prefix in db_service_prefixes or []:
+                    db_service_name, *_ = self.parse_db_service_prefix(
+                        db_service_prefix
+                    )
+                    dialect = self._get_db_dialect(db_service_name)
+                    source_table_name = self._clean_table_name(sql_table_name, dialect)
+                    self._parsed_views[view.name] = source_table_name
+
+                    # Extract column lineage
+                    column_lineage = self._extract_column_lineage(view)
+
+                    # View to the source is only there if we are informing the dbServiceNames
                     yield self.build_lineage_request(
                         source=source_table_name,
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         to_entity=self._view_data_model,
+                        column_lineage=column_lineage,
                     )
 
             elif view.derived_table:
                 sql_query = view.derived_table.sql
                 if not sql_query:
                     return
-                for db_service_name in db_service_names or []:
-                    db_service = self.metadata.get_by_name(
-                        DatabaseService, db_service_name
-                    )
-
-                    lineage_parser = LineageParser(
-                        sql_query,
-                        ConnectionTypeDialectMapper.dialect_of(
-                            db_service.connection.config.type.value
-                        ),
-                        timeout_seconds=30,
-                    )
-                    if lineage_parser.source_tables:
-                        for from_table_name in lineage_parser.source_tables:
-                            yield self.build_lineage_request(
-                                source=str(from_table_name),
-                                db_service_name=db_service_name,
-                                to_entity=self._view_data_model,
-                            )
+                if find_derived_references(sql_query):
+                    sql_query = self.replace_derived_references(sql_query)
+                    # If we still have derived references, we cannot process the view
+                    if view_references := find_derived_references(sql_query):
+                        self._add_dependency_edge(view.name, view_references)
+                        logger.warning(
+                            f"Not all references are replaced for view [{view.name}]. Parsing it later."
+                        )
+                        return
+                logger.debug(f"Processing view [{view.name}] with SQL: \n[{sql_query}]")
+                yield from self._build_lineage_for_view(view.name, sql_query)
+                if self._unparsed_views:
+                    self.build_lineage_for_unparsed_views()
 
         except Exception as err:
             yield Either(
@@ -615,10 +962,63 @@ class LookerSource(DashboardServiceSource):
                 )
             )
 
+    def _build_lineage_for_view(
+        self, view_name: str, sql_query: str
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Parse the SQL query and build lineage for the view.
+        """
+        for db_service_prefix in self.get_db_service_prefixes() or []:
+            db_service_name, *_ = self.parse_db_service_prefix(db_service_prefix)
+            lineage_parser = LineageParser(
+                f"create view {view_name} as {sql_query}",
+                self._get_db_dialect(db_service_name),
+                timeout_seconds=30,
+            )
+            if lineage_parser.source_tables:
+                self._parsed_views[view_name] = sql_query
+                for from_table_name in lineage_parser.source_tables:
+                    # Process column level lineage
+                    column_lineage = []
+                    for column_tuple in lineage_parser.column_lineage or []:
+                        if (
+                            column_tuple
+                            and hasattr(column_tuple[0], "parent")
+                            and column_tuple[0].parent == from_table_name
+                        ):
+                            column_lineage.append(
+                                (
+                                    (
+                                        column_tuple[0].raw_name
+                                        if hasattr(column_tuple[0], "raw_name")
+                                        else column_tuple[0]
+                                    ),
+                                    (
+                                        column_tuple[-1].raw_name
+                                        if hasattr(column_tuple[-1], "raw_name")
+                                        else column_tuple[-1]
+                                    ),
+                                )
+                            )
+                    yield self.build_lineage_request(
+                        source=str(from_table_name),
+                        db_service_prefix=db_service_prefix,
+                        to_entity=self._view_data_model,
+                        column_lineage=column_lineage if column_lineage else None,
+                    )
+
+    def _get_db_dialect(self, db_service_name) -> Dialect:
+        db_service = self.metadata.get_by_name(DatabaseService, db_service_name)
+        return ConnectionTypeDialectMapper.dialect_of(
+            db_service.connection.config.type.value
+        )
+
     def get_dashboards_list(self) -> List[DashboardBase]:
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         try:
             return list(
                 self.client.all_dashboards(fields=",".join(LIST_DASHBOARD_FIELDS))
@@ -660,10 +1060,14 @@ class LookerSource(DashboardServiceSource):
             Optional[EntityReference]
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.user_id is not None:
                 dashboard_owner = self.client.user(dashboard_details.user_id)
-                return self.metadata.get_reference_by_email(dashboard_owner.email)
-
+                if dashboard_owner.email:
+                    return self.metadata.get_reference_by_email(
+                        dashboard_owner.email.lower()
+                    )
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(f"Could not fetch owner data due to {err}")
@@ -679,9 +1083,11 @@ class LookerSource(DashboardServiceSource):
         dashboard_request = CreateDashboardRequest(
             name=EntityName(clean_dashboard_name(dashboard_details.id)),
             displayName=dashboard_details.title,
-            description=Markdown(dashboard_details.description)
-            if dashboard_details.description
-            else None,
+            description=(
+                Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None
+            ),
             charts=[
                 FullyQualifiedEntityName(
                     fqn.build(
@@ -718,7 +1124,7 @@ class LookerSource(DashboardServiceSource):
         return None
 
     @staticmethod
-    def _clean_table_name(table_name: str) -> str:
+    def _clean_table_name(table_name: str, dialect: Dialect = Dialect.ANSI) -> str:
         """
         sql_table_names might be renamed when defining
         an explore. E.g., customers as cust
@@ -726,7 +1132,10 @@ class LookerSource(DashboardServiceSource):
         :return: clean table name
         """
 
-        return table_name.lower().split(" as ")[0].strip()
+        clean_table_name = table_name.lower().split(" as ")[0].strip()
+        if dialect == Dialect.BIGQUERY:
+            clean_table_name = clean_table_name.strip("`")
+        return clean_table_name
 
     @staticmethod
     def _render_table_name(table_name: str) -> str:
@@ -801,7 +1210,9 @@ class LookerSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: LookerDashboard, _: str
+        self,
+        dashboard_details: LookerDashboard,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -853,11 +1264,61 @@ class LookerSource(DashboardServiceSource):
                 )
             )
 
+    def _process_and_validate_column_lineage(
+        self,
+        column_lineage: List[Tuple[Column, Column]],
+        from_entity: Table,
+        to_entity: Union[Dashboard, DashboardDataModel],
+    ) -> List[ColumnLineage]:
+        """
+        Process and validate column lineage
+        """
+        processed_column_lineage = []
+        if column_lineage:
+            for column_tuple in column_lineage or []:
+                try:
+                    if len(column_tuple) < 2:
+                        logger.debug(f"Skipping invalid column tuple: {column_tuple}")
+                        continue
+
+                    source_col = column_tuple[0]
+                    target_col = column_tuple[-1]
+
+                    if not source_col or not target_col:
+                        logger.debug(
+                            f"Skipping column tuple with empty values: source={source_col}, "
+                            f"target={target_col}, to_entity={to_entity.name}"
+                        )
+                        continue
+
+                    from_column = get_column_fqn(
+                        table_entity=from_entity, column=str(target_col)
+                    )
+                    to_column = self._get_data_model_column_fqn(
+                        data_model_entity=to_entity,
+                        column=str(source_col),
+                    )
+                    if from_column and to_column:
+                        processed_column_lineage.append(
+                            ColumnLineage(
+                                fromColumns=[from_column],
+                                toColumn=to_column,
+                            )
+                        )
+                except Exception as err:
+                    logger.warning(
+                        f"Error processing column lineage {column_tuple}: {err}"
+                    )
+                    logger.debug(traceback.format_exc())
+                    continue
+        return processed_column_lineage
+
     def build_lineage_request(
         self,
         source: str,
-        db_service_name: str,
+        db_service_prefix: str,
         to_entity: Union[Dashboard, DashboardDataModel],
+        column_lineage: Optional[List[Tuple[Column, Column]]] = None,
     ) -> Optional[Either[AddLineageRequest]]:
         """
         Once we have a list of origin data sources, check their components
@@ -867,20 +1328,48 @@ class LookerSource(DashboardServiceSource):
 
         Args:
             source: table name from the source list
-            db_service_name: name of the service from the config
+            db_service_prefix: db service prefix from the config
             to_entity: Dashboard Entity being used
         """
+        logger.debug(f"Building lineage request for {source} to {to_entity.name}")
 
         source_elements = fqn.split_table_name(table_name=source)
 
+        (
+            db_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         for database_name in [source_elements["database"], None]:
+            if (
+                (
+                    prefix_database_name
+                    and database_name
+                    and prefix_database_name.lower() != database_name.lower()
+                )
+                or (
+                    prefix_schema_name
+                    and source_elements["database_schema"]
+                    and prefix_schema_name.lower()
+                    != source_elements["database_schema"].lower()
+                )
+                or (
+                    prefix_table_name
+                    and source_elements["table"]
+                    and prefix_table_name.lower() != source_elements["table"].lower()
+                )
+            ):
+                continue
+
             from_fqn = fqn.build(
                 self.metadata,
                 entity_type=Table,
                 service_name=db_service_name,
-                database_name=database_name,
-                schema_name=source_elements["database_schema"],
-                table_name=source_elements["table"],
+                database_name=prefix_database_name or database_name,
+                schema_name=prefix_schema_name or source_elements["database_schema"],
+                table_name=prefix_table_name or source_elements["table"],
             )
 
             from_entity: Table = self.metadata.get_by_name(
@@ -893,8 +1382,15 @@ class LookerSource(DashboardServiceSource):
                     self._added_lineage[from_entity.id.root] = []
                 if to_entity.id.root not in self._added_lineage[from_entity.id.root]:
                     self._added_lineage[from_entity.id.root].append(to_entity.id.root)
+                    processed_column_lineage = (
+                        self._process_and_validate_column_lineage(
+                            column_lineage, from_entity, to_entity
+                        )
+                    )
                     return self._get_add_lineage_request(
-                        to_entity=to_entity, from_entity=from_entity
+                        to_entity=to_entity,
+                        from_entity=from_entity,
+                        column_lineage=processed_column_lineage,
                     )
 
         return None
@@ -919,17 +1415,19 @@ class LookerSource(DashboardServiceSource):
                     continue
 
                 description = self.build_chart_description(chart)
+                if chart.query is not None:
+                    source_url = chart.query.share_url
+                elif getattr(chart.result_maker, "query", None) is not None:
+                    source_url = chart.result_maker.query.share_url
+                else:
+                    source_url = f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
                 yield Either(
                     right=CreateChartRequest(
                         name=EntityName(chart.id),
                         displayName=chart.title or chart.id,
                         description=Markdown(description) if description else None,
                         chartType=get_standard_chart_type(chart.type).value,
-                        sourceUrl=SourceUrl(chart.query.share_url)
-                        if chart.query is not None
-                        else SourceUrl(
-                            f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}"
-                        ),
+                        sourceUrl=SourceUrl(source_url),
                         service=self.context.get().dashboard_service,
                     )
                 )
@@ -1071,3 +1569,7 @@ class LookerSource(DashboardServiceSource):
                     stackTrace=traceback.format_exc(),
                 )
             )
+
+    def close(self):
+        self.metadata.compute_percentile(Dashboard, self.today)
+        self.metadata.close()