openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/hex/warehouse_queries.py

@@ -0,0 +1,225 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Hex-specific query templates for fetching queries from various data warehouses
+"""
+
+import textwrap
+from enum import Enum
+from typing import Dict
+
+
+class WarehouseType(Enum):
+    """Supported warehouse types for Hex lineage"""
+
+    SNOWFLAKE = "snowflake"
+    BIGQUERY = "bigquery"
+    DATABRICKS = "databricks"
+    REDSHIFT = "redshift"
+    MYSQL = "mysql"
+    ATHENA = "athena"
+    TRINO = "trino"
+    PRESTO = "presto"
+    CLICKHOUSE = "clickhouse"
+
+
+# Snowflake query to fetch Hex-originated queries
+HEX_SNOWFLAKE_QUERY = textwrap.dedent(
+    """
+    SELECT query_text,query_id, user_name, start_time
+    FROM {account_usage}.QUERY_HISTORY
+    WHERE query_text ILIKE '%hex%'
+    ORDER BY start_time DESC
+    LIMIT {limit};
+    """
+)
+
+# BigQuery query to fetch Hex-originated queries
+HEX_BIGQUERY_QUERY = textwrap.dedent(
+    """
+    SELECT 
+        query as query_text,
+        user_email as user_name,
+        project_id as database_name,
+        CAST(NULL AS STRING) as schema_name,
+        start_time,
+        end_time,
+        total_slot_ms as duration
+    FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
+    WHERE 
+        query LIKE '%hex%'
+        AND creation_time >= TIMESTAMP('{start_time}')
+        AND creation_time <= TIMESTAMP('{end_time}')
+    ORDER BY creation_time DESC
+    LIMIT {limit}
+    """
+)
+
+# Databricks query to fetch Hex-originated queries
+HEX_DATABRICKS_QUERY = textwrap.dedent(
+    """
+    SELECT
+      statement_text as query_text
+    FROM system.query.history
+    WHERE 
+      LOWER(statement_text) LIKE '%hex%'
+      AND start_time >= '{start_time}'
+      AND start_time <= '{end_time}'
+    ORDER BY start_time DESC
+    LIMIT {limit}
+    """
+)
+
+# Redshift query to fetch Hex-originated queries
+HEX_REDSHIFT_QUERY = textwrap.dedent(
+    """
+    SELECT
+      q.querytxt as query_text,
+      u.usename as user_name,
+      trim(q.database) as database_name,
+      q.starttime as start_time,
+      q.endtime as end_time,
+      DATEDIFF(milliseconds, q.starttime, q.endtime) as duration_ms,
+      q.query as query_id
+    FROM stl_query q
+    LEFT JOIN pg_user u ON q.userid = u.usesysid
+    WHERE 
+      LOWER(q.querytxt) LIKE '%hex%'
+      AND q.starttime >= '{start_time}'
+      AND q.starttime <= '{end_time}'
+      AND q.querytxt NOT LIKE 'COMMIT%'
+      AND q.querytxt NOT LIKE 'BEGIN%'
+      AND q.querytxt NOT LIKE 'ROLLBACK%'
+      AND q.querytxt NOT LIKE 'SET %'
+      AND q.querytxt NOT LIKE 'SHOW %'
+      AND q.querytxt NOT LIKE '%pg_catalog%'
+      AND q.querytxt NOT LIKE '%information_schema%'
+      AND q.querytxt NOT LIKE '%stl_query%'
+    ORDER BY q.starttime DESC
+    LIMIT {limit}
+    """
+)
+
+
+# MySQL query to fetch Hex-originated queries
+# Note: This requires either:
+# 1. performance_schema with statement history enabled
+# 2. general_log table enabled (SET GLOBAL general_log = 'ON'; SET GLOBAL log_output = 'TABLE';)
+# 3. Custom query logging solution
+HEX_MYSQL_QUERY = textwrap.dedent(
+    """
+    SELECT 
+       CONVERT(argument USING utf8) as query_text,
+       user_host as user_name,
+       event_time as start_time
+FROM mysql.general_log
+WHERE argument LIKE '%hex%'
+LIMIT {limit};
+"""
+)
+
+
+# Athena query to fetch Hex-originated queries
+HEX_ATHENA_QUERY = textwrap.dedent(
+    """
+    SELECT
+      query_string as query_text,
+      principal as user_name,
+      catalog as database_name,
+      schema as schema_name,
+      event_time as start_time,
+      event_time as end_time,
+      query_execution_time_in_millis as duration
+    FROM aws_athena_query_history
+    WHERE query_string LIKE '%Hex query metadata%'
+      AND (
+        query_string LIKE '%"context": "SCHEDULED_RUN"%' 
+        OR query_string LIKE '%"context": "APP_VIEW"%'
+      )
+      AND event_time BETWEEN '{start_time}' AND '{end_time}'
+    LIMIT {limit}
+    """
+)
+
+# Trino/Presto query to fetch Hex-originated queries
+HEX_TRINO_QUERY = textwrap.dedent(
+    """
+    SELECT
+      query as query_text,
+      user as user_name,
+      catalog as database_name,
+      schema as schema_name,
+      created as start_time,
+      end as end_time,
+      query_wall_time_ms as duration
+    FROM system.runtime.queries
+    WHERE query LIKE '%Hex query metadata%'
+      AND (
+        query LIKE '%"context": "SCHEDULED_RUN"%' 
+        OR query LIKE '%"context": "APP_VIEW"%'
+      )
+      AND created BETWEEN timestamp '{start_time}' AND timestamp '{end_time}'
+    LIMIT {limit}
+    """
+)
+
+# ClickHouse query to fetch Hex-originated queries
+HEX_CLICKHOUSE_QUERY = textwrap.dedent(
+    """
+    SELECT
+      query as query_text,
+      user as user_name,
+      current_database as database_name,
+      '' as schema_name,
+      event_time as start_time,
+      event_time as end_time,
+      query_duration_ms as duration
+    FROM system.query_log
+    WHERE query LIKE '%Hex query metadata%'
+      AND event_time BETWEEN '{start_time}' AND '{end_time}'
+      AND type = 'QueryFinish'
+    LIMIT {limit}
+    """
+)
+
+
+# Mapping of warehouse types to their query templates
+HEX_WAREHOUSE_QUERIES: Dict[WarehouseType, str] = {
+    WarehouseType.SNOWFLAKE: HEX_SNOWFLAKE_QUERY,
+    WarehouseType.BIGQUERY: HEX_BIGQUERY_QUERY,
+    WarehouseType.DATABRICKS: HEX_DATABRICKS_QUERY,
+    WarehouseType.REDSHIFT: HEX_REDSHIFT_QUERY,
+    WarehouseType.MYSQL: HEX_MYSQL_QUERY,
+    WarehouseType.ATHENA: HEX_ATHENA_QUERY,
+    # WarehouseType.TRINO: HEX_TRINO_QUERY,
+    # WarehouseType.PRESTO: HEX_TRINO_QUERY,  # Same as Trino
+    WarehouseType.CLICKHOUSE: HEX_CLICKHOUSE_QUERY,
+}
+
+
+def get_hex_query_template(warehouse_type: str) -> str:
+    """
+    Get the Hex query template for a specific warehouse type
+
+    Args:
+        warehouse_type: Type of the warehouse (e.g., "snowflake", "bigquery")
+
+    Returns:
+        Query template string for the warehouse
+
+    Raises:
+        ValueError: If warehouse type is not supported
+    """
+    try:
+        warehouse = WarehouseType(warehouse_type.lower())
+        return HEX_WAREHOUSE_QUERIES.get(warehouse)
+    except ValueError:
+        raise ValueError(f"Unsupported warehouse type: {warehouse_type}")

metadata/ingestion/source/dashboard/looker/bulk_parser.py

@@ -50,9 +50,15 @@ class BulkLkmlParser(metaclass=Singleton):
     Approach:
     When we parse, we parse all files *.view.lkml to get all view and cached them. It can speed up the process and avoid
     infinity loop when parsing includes.
+
+    Supports multiple readers to aggregate views from multiple repositories.
     """
 
-    def __init__(self, reader: LocalReader):
+    def __init__(
+        self,
+        reader: LocalReader,
+        additional_readers: Optional[List[LocalReader]] = None,
+    ):
         self._views_cache: Dict[ViewName, LookMlView] = {}
         self._visited_files: Dict[Includes, List[Includes]] = {}
 
@@ -61,34 +67,45 @@ class BulkLkmlParser(metaclass=Singleton):
         self.parsed_view: Dict[str, List[Includes]] = {}
 
         self.reader = reader
-        self.__parse_all_views()
+        self.additional_readers = additional_readers or []
+
+        # Parse views from the primary reader
+        self.__parse_all_views(self.reader)
+
+        # Parse views from additional readers
+        for additional_reader in self.additional_readers:
+            self.__parse_all_views(additional_reader)
 
-    def __parse_all_views(self):
-        file_paths = self.reader.get_local_files(search_key=".view.lkml")
+    def __parse_all_views(self, reader: LocalReader):
+        file_paths = reader.get_local_files(search_key=".view.lkml")
         for _path in file_paths:
-            file = self._read_file(Includes(_path))
-            lkml_file = LkmlFile.model_validate(lkml.load(file))
-            self.parsed_files[Includes(_path)] = file
-            for view in lkml_file.views:
-                view.source_file = _path
-                self._views_cache[view.name] = view
-
-    def _read_file(self, path: Includes) -> str:
+            try:
+                file = self._read_file(Includes(_path), reader)
+                lkml_file = LkmlFile.model_validate(lkml.load(file))
+                self.parsed_files[Includes(_path)] = file
+                for view in lkml_file.views:
+                    view.source_file = _path
+                    self._views_cache[view.name] = view
+            except Exception as err:
+                logger.debug(f"Error parsing file {_path}: {err}")
+
+    def _read_file(self, path: Includes, reader: Optional[LocalReader] = None) -> str:
         """
         Read the LookML file
         """
+        reader_to_use = reader or self.reader
         suffixes = Path(path).suffixes
 
         # Check if any suffix is in our extension list
         if not set(suffixes).intersection(set(EXTENSIONS)):
             for suffix in EXTENSIONS:
                 try:
-                    return self.reader.read(path + suffix)
+                    return reader_to_use.read(path + suffix)
                 except ReadException as err:
                     logger.debug(f"Error trying to read the file [{path}]: {err}")
 
         else:
-            return self.reader.read(path)
+            return reader_to_use.read(path)
 
         raise ReadException(f"Error trying to read the file [{path}]")
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -69,6 +69,7 @@ from metadata.generated.schema.entity.data.dashboardDataModel import (
 )
 from metadata.generated.schema.entity.data.table import Column, Table
 from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
+    LocalRepositoryPath,
     LookerConnection,
     NoGitCredentials,
 )
@@ -210,10 +211,11 @@ class LookerSource(DashboardServiceSource):
         self.today = datetime.now().strftime("%Y-%m-%d")
 
         self._explores_cache = {}
+        self._views_cache = {}
         self._repo_credentials: Optional[ReadersCredentials] = None
         self._reader_class: Optional[Type[Reader]] = None
         self._project_parsers: Optional[Dict[str, BulkLkmlParser]] = None
-        self._main_lookml_repo: Optional[LookMLRepo] = None
+        self._main_lookml_repos: Optional[List[LookMLRepo]] = None
         self._main__lookml_manifest: Optional[LookMLManifest] = None
         self._view_data_model: Optional[DashboardDataModel] = None
 
@@ -243,59 +245,105 @@ class LookerSource(DashboardServiceSource):
         credentials: Optional[
             Union[
                 NoGitCredentials,
+                LocalRepositoryPath,
                 GitHubCredentials,
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
         ],
-    ) -> "LookMLRepo":
-        repo_name = (
-            f"{credentials.repositoryOwner.root}/{credentials.repositoryName.root}"
-        )
-        repo_path = f"{REPO_TMP_LOCAL_PATH}/{credentials.repositoryName.root}"
-        _clone_repo(
-            repo_name,
-            repo_path,
-            credentials,
-            overwrite=True,
-        )
-        return LookMLRepo(name=repo_name, path=repo_path)
+    ) -> List["LookMLRepo"]:
+        repos = []
+        if isinstance(credentials, LocalRepositoryPath):
+            # For local repository path, use the path directly without cloning
+            local_path = Path(credentials.root)
+            repo_name = local_path.name
+            repos.append(LookMLRepo(name=repo_name, path=str(local_path)))
+        elif isinstance(
+            credentials, (GitHubCredentials, BitBucketCredentials, GitlabCredentials)
+        ):
+            # Support comma-separated repository names
+            repository_names = [
+                name.strip()
+                for name in credentials.repositoryName.root.split(",")
+                if name.strip()
+            ]
+
+            for repo_name_only in repository_names:
+                repo_name = f"{credentials.repositoryOwner.root}/{repo_name_only}"
+                repo_path = f"{REPO_TMP_LOCAL_PATH}/{repo_name_only}"
+
+                single_repo_creds = copy.deepcopy(credentials)
+                single_repo_creds.repositoryName.root = repo_name_only
+
+                _clone_repo(
+                    repo_name,
+                    repo_path,
+                    single_repo_creds,
+                    overwrite=True,
+                )
+                repos.append(LookMLRepo(name=repo_name, path=repo_path))
+        else:
+            # For NoGitCredentials or other unsupported types
+            raise ValueError(f"Unsupported credential type: {type(credentials)}")
+
+        return repos
 
     def __read_manifest(
         self,
         credentials: Optional[
             Union[
                 NoGitCredentials,
+                LocalRepositoryPath,
                 GitHubCredentials,
                 BitBucketCredentials,
                 GitlabCredentials,
             ]
         ],
+        repo: LookMLRepo,
         path="manifest.lkml",
     ) -> Optional[LookMLManifest]:
-        file_path = f"{self._main_lookml_repo.path}/{path}"
-        if not os.path.isfile(file_path):
+        file_path = Path(repo.path) / path
+        if not file_path.is_file():
+            if isinstance(credentials, LocalRepositoryPath):
+                logger.warning(
+                    f"Manifest file '{path}' not found in local repository at {file_path}. "
+                    f"Ensure the manifest file exists in your local LookML repository."
+                )
             return None
+
         with open(file_path, "r", encoding="utf-8") as fle:
             manifest = LookMLManifest.model_validate(lkml.load(fle))
             if manifest and manifest.remote_dependency:
                 remote_name = manifest.remote_dependency["name"]
                 remote_git_url = manifest.remote_dependency["url"]
 
-                url_parsed = giturlparse.parse(remote_git_url)
-                _clone_repo(
-                    f"{url_parsed.owner}/{url_parsed.repo}",  # pylint: disable=E1101
-                    f"{self._main_lookml_repo.path}/{IMPORTED_PROJECTS_DIR}/{remote_name}",
-                    credentials,
-                )
+                if isinstance(credentials, LocalRepositoryPath):
+                    # For local repository path, warn about remote dependencies
+                    logger.warning(
+                        f"Remote dependency '{remote_name}' found in manifest. "
+                        f"When using localRepositoryPath, remote dependencies are not automatically fetched. "
+                        f"If needed, manually place the dependency in the '{IMPORTED_PROJECTS_DIR}/{remote_name}' directory within your local repository."
+                    )
+                else:
+                    # For remote repositories, clone the dependency as before
+                    url_parsed = giturlparse.parse(remote_git_url)
+                    _clone_repo(
+                        f"{url_parsed.owner}/{url_parsed.repo}",  # pylint: disable=E1101
+                        f"{repo.path}/{IMPORTED_PROJECTS_DIR}/{remote_name}",
+                        credentials,
+                    )
 
             return manifest
 
     def prepare(self):
         if self.service_connection.gitCredentials:
             credentials = self.service_connection.gitCredentials
-            self._main_lookml_repo = self.__init_repo(credentials)
-            self._main__lookml_manifest = self.__read_manifest(credentials)
+            self._main_lookml_repos = self.__init_repo(credentials)
+            if self._main_lookml_repos:
+                # Read manifest from the first repository (primary repository)
+                self._main__lookml_manifest = self.__read_manifest(
+                    credentials, self._main_lookml_repos[0]
+                )
 
     @property
     def parser(self) -> Optional[Dict[str, BulkLkmlParser]]:
@@ -317,15 +365,27 @@ class LookerSource(DashboardServiceSource):
         and can be accessed with the same token. If we have
         any errors obtaining the git project information, we will default
         to the incoming GitHub Credentials.
+
+        Since BulkLkmlParser uses Singleton pattern, we create ONE parser per project
+        that aggregates views from all repositories.
         """
-        if self.repository_credentials:
+        if self.repository_credentials and self._main_lookml_repos:
             all_projects: Set[str] = {model.project_name for model in all_lookml_models}
-            self._project_parsers: Dict[str, BulkLkmlParser] = {
-                project_name: BulkLkmlParser(
-                    reader=self.reader(Path(self._main_lookml_repo.path))
+            self._project_parsers: Dict[str, BulkLkmlParser] = {}
+
+            # Create readers for all repositories
+            primary_reader = self.reader(Path(self._main_lookml_repos[0].path))
+            additional_readers = [
+                self.reader(Path(repo.path)) for repo in self._main_lookml_repos[1:]
+            ]
+
+            # For each project, create a single parser with all readers
+            for project_name in all_projects:
+                parser = BulkLkmlParser(
+                    reader=primary_reader, additional_readers=additional_readers
                 )
-                for project_name in all_projects
-            }
+                self._project_parsers[project_name] = parser
+
             logger.info(f"We found the following parsers:\n {self._project_parsers}")
 
     def get_lookml_project_credentials(self, project_name: str) -> ReadersCredentials:
@@ -362,8 +422,13 @@ class LookerSource(DashboardServiceSource):
         We either get GitHubCredentials or `NoGitHubCredentials`
         """
         if not self._repo_credentials:
-            if self.service_connection.gitCredentials and isinstance(
-                self.service_connection.gitCredentials, get_args(ReadersCredentials)
+            if self.service_connection.gitCredentials and (
+                isinstance(
+                    self.service_connection.gitCredentials, get_args(ReadersCredentials)
+                )
+                or isinstance(
+                    self.service_connection.gitCredentials, LocalRepositoryPath
+                )
             ):
                 self._repo_credentials = self.service_connection.gitCredentials
 
@@ -504,6 +569,13 @@ class LookerSource(DashboardServiceSource):
                 # We can get VIEWs from the JOINs to know the dependencies
                 # We will only try and fetch if we have the credentials
                 if self.repository_credentials:
+                    logger.info(
+                        f"Repository credentials are present, processing views of explore model {datamodel_name}"
+                    )
+                    if model.joins:
+                        logger.info(
+                            f"Joins are present, processing views of explore model {datamodel_name}"
+                        )
                     for view in model.joins:
                         if filter_by_datamodel(
                             self.source_config.dataModelFilterPattern, view.name
@@ -517,6 +589,9 @@ class LookerSource(DashboardServiceSource):
                             view_name=ViewName(view_name), explore=model
                         )
                     if model.view_name:
+                        logger.info(
+                            f"View name is present, processing view {model.view_name} of explore model {datamodel_name}"
+                        )
                         yield from self._process_view(
                             view_name=ViewName(model.view_name), explore=model
                         )
@@ -605,6 +680,7 @@ class LookerSource(DashboardServiceSource):
                 )
                 yield Either(right=data_model_request)
                 self._view_data_model = self._build_data_model(datamodel_view_name)
+                self._views_cache[view.name] = self._view_data_model
                 self.register_record_datamodel(datamodel_request=data_model_request)
                 yield from self.add_view_lineage(view, explore)
             else:
@@ -816,6 +892,25 @@ class LookerSource(DashboardServiceSource):
                     " while processing view lineage."
                 )
 
+            # Handle view-to-view lineage via extends
+            if view.extends__all:
+                for extended_views_list in view.extends__all:
+                    for extended_view_name in extended_views_list:
+                        extended_view_model = self._views_cache.get(extended_view_name)
+                        if extended_view_model:
+                            logger.debug(
+                                f"Building lineage from extended view {extended_view_name} to view {self._view_data_model.name}"
+                            )
+                            yield self._get_add_lineage_request(
+                                from_entity=extended_view_model,
+                                to_entity=self._view_data_model,
+                                column_lineage=[],
+                            )
+                        else:
+                            logger.debug(
+                                f"Extended view [{extended_view_name}] not found in cache for view [{view.name}]"
+                            )
+
             db_service_prefixes = self.get_db_service_prefixes()
 
             if view.sql_table_name:
@@ -922,6 +1017,8 @@ class LookerSource(DashboardServiceSource):
         """
         Get List of all dashboards
         """
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         try:
             return list(
                 self.client.all_dashboards(fields=",".join(LIST_DASHBOARD_FIELDS))
@@ -963,6 +1060,8 @@ class LookerSource(DashboardServiceSource):
             Optional[EntityReference]
         """
         try:
+            if not self.source_config.includeOwners:
+                return None
             if dashboard_details.user_id is not None:
                 dashboard_owner = self.client.user(dashboard_details.user_id)
                 if dashboard_owner.email:
@@ -1473,4 +1572,4 @@ class LookerSource(DashboardServiceSource):
 
     def close(self):
         self.metadata.compute_percentile(Dashboard, self.today)
-        self.metadata.close()
+        self.metadata.close()

metadata/ingestion/source/dashboard/looker/models.py

@@ -53,6 +53,9 @@ class LookMlView(BaseModel):
         None, description="To track lineage with the source"
     )
     tags: Optional[List[str]] = Field(None, description="Tags for the view")
+    extends__all: Optional[List[List[str]]] = Field(
+        None, alias="extends__all", description="List of views this view extends"
+    )
 
 
 class LkmlFile(BaseModel):

metadata/ingestion/source/dashboard/looker/utils.py

@@ -36,6 +36,15 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 
 
+def _extract_hostname(git_host_url) -> str:
+    """Extract hostname from git host URL by removing protocol prefix"""
+    # Handle both string and AnyUrl/HttpUrl objects from Pydantic
+    url_str = str(git_host_url)
+    # Remove protocol and trailing slash
+    hostname = url_str.replace("https://", "").replace("http://", "").rstrip("/")
+    return hostname
+
+
 def _clone_repo(
     repo_name: str,
     path: str,
@@ -57,12 +66,18 @@ def _clone_repo(
         url = None
         allow_unsafe_protocols = False
         if isinstance(credential, GitHubCredentials):
-            url = f"https://x-oauth-basic:{credential.token.root.get_secret_value()}@github.com/{repo_name}.git"
+            git_host = credential.gitHostURL or "https://github.com"
+            hostname = _extract_hostname(git_host)
+            url = f"https://x-oauth-basic:{credential.token.root.get_secret_value()}@{hostname}/{repo_name}.git"
         elif isinstance(credential, BitBucketCredentials):
-            url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@bitbucket.org/{repo_name}.git"
+            git_host = credential.gitHostURL or "https://bitbucket.org"
+            hostname = _extract_hostname(git_host)
+            url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@{hostname}/{repo_name}.git"
             allow_unsafe_protocols = True
         elif isinstance(credential, GitlabCredentials):
-            url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@gitlab.com/{repo_name}.git"
+            git_host = credential.gitHostURL or "https://gitlab.com"
+            hostname = _extract_hostname(git_host)
+            url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@{hostname}/{repo_name}.git"
 
         assert url is not None