openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.1.0__py3-none-any.whl

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-06-24T04:55:53+00:00
+#   timestamp: 2025-07-01T14:19:32+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/api/rest/metadata.py

@@ -300,38 +300,79 @@ class RestSource(ApiServiceSource):
             logger.warning(f"Error while parsing response schema: {err}")
         return None
 
-    def process_schema_fields(self, schema_ref: str) -> Optional[List[FieldModel]]:
+    def process_schema_fields(
+        self, schema_ref: str, parent_refs: Optional[List[str]] = None
+    ) -> Optional[List[FieldModel]]:
         try:
+            if parent_refs is None:
+                parent_refs = []
             schema_name = schema_ref.split("/")[-1]
             schema_fields = (
                 self.json_response.get("components").get("schemas").get(schema_name)
             )
-
+            parent_refs.append(schema_ref)
             fetched_fields = []
             for key, val in schema_fields.get("properties", {}).items():
                 dtype = val.get("type")
                 if dtype:
+                    dtype = "INT" if dtype.upper() == "INTEGER" else dtype
                     parsed_dtype = (
                         DataTypeTopic[dtype.upper()]
                         if dtype.upper() in DataTypeTopic.__members__
                         else DataTypeTopic.UNKNOWN
                     )
-                    fetched_fields.append(FieldModel(name=key, dataType=parsed_dtype))
+                    children = None
+                    if parsed_dtype.value == DataTypeTopic.ARRAY.value:
+                        # If field of array type then parse children
+                        children_ref = val.get("items", {}).get("$ref")
+                        if children_ref:
+                            # check infinite recursion by checking pre-processed schemas(parent_refs)
+                            if children_ref not in parent_refs:
+                                logger.debug(
+                                    f"Processing array fields inside schema: {children_ref}"
+                                )
+                                children = self.process_schema_fields(
+                                    children_ref, parent_refs
+                                )
+                                logger.debug(
+                                    f"Completed processing array fields inside schema: {children_ref}"
+                                )
+                            else:
+                                logger.debug(
+                                    f"Skipping array fields inside schema: {children_ref} to avoid infinite recursion"
+                                )
+                    fetched_fields.append(
+                        FieldModel(name=key, dataType=parsed_dtype, children=children)
+                    )
                 else:
                     # If type of field is not defined then check for sub-schema
                     # Check if it's `object` type field
-                    # check infinite recrusrion by comparing with parent(schema_ref)
-                    object_children = None
-                    if val.get("$ref") and val.get("$ref") != schema_ref:
-                        object_children = self.process_schema_fields(val.get("$ref"))
+                    children = None
+                    if val.get("$ref"):
+                        # check infinite recursion by checking pre-processed schemas(parent_refs)
+                        if val.get("$ref") not in parent_refs:
+                            children = self.process_schema_fields(
+                                val.get("$ref"), parent_refs
+                            )
+                        else:
+                            logger.debug(
+                                f"Skipping object fields inside schema: {val.get('$ref')} to avoid infinite recursion"
+                            )
                     fetched_fields.append(
                         FieldModel(
                             name=key,
                             dataType=DataTypeTopic.UNKNOWN,
-                            children=object_children,
+                            children=children,
                         )
                     )
+            if parent_refs and (schema_ref in parent_refs):
+                parent_refs.pop()
             return fetched_fields
         except Exception as err:
             logger.warning(f"Error while processing schema fields: {err}")
+            if parent_refs and (schema_ref in parent_refs):
+                parent_refs.pop()
+                logger.debug(
+                    f"Popping {schema_ref} from parent_refs due to processing error"
+                )
         return None

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -12,6 +12,7 @@
 REST Auth & Client for QlikCloud
 """
 import json
+import re
 import traceback
 from typing import Dict, Iterable, List, Optional
 
@@ -23,12 +24,14 @@ from metadata.ingestion.source.dashboard.qlikcloud.constants import (
     APP_LOADMODEL_REQ,
     CREATE_SHEET_SESSION,
     GET_LOADMODEL_LAYOUT,
+    GET_SCRIPT,
     GET_SHEET_LAYOUT,
     OPEN_DOC_REQ,
 )
 from metadata.ingestion.source.dashboard.qlikcloud.models import (
     QlikApp,
     QlikAppResponse,
+    QlikScriptResult,
     QlikSpace,
     QlikSpaceResponse,
 )
@@ -173,12 +176,18 @@ class QlikCloudClient:
             models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True)
             data_models = QlikDataModelResult(**models)
             layout = data_models.result.qLayout
+            parsed_datamodels = []
             if isinstance(layout, list):
                 tables = []
                 for layout in data_models.result.qLayout:
                     tables.extend(layout.value.tables)
-                return tables
-            return layout.tables
+                parsed_datamodels.extend(tables)
+            else:
+                parsed_datamodels.extend(layout.tables)
+            script_tables = self.get_script_tables()
+            if script_tables:
+                parsed_datamodels.extend(script_tables)
+            return parsed_datamodels
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the dashboard datamodels")
@@ -204,3 +213,26 @@ class QlikCloudClient:
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the space list")
+
+    def get_script_tables(self) -> Optional[List[QlikTable]]:
+        """Get script tables from the dashboard script"""
+        script_tables = []
+        try:
+            script_response = self._websocket_send_request(GET_SCRIPT, response=True)
+            script_result = QlikScriptResult(**script_response)
+            if script_result.result.qScript:
+                script_value = script_result.result.qScript
+                matches = re.findall(
+                    r'FROM\s+["\']?([a-zA-Z0-9_.]+)["\']?', script_value, re.IGNORECASE
+                )
+                if isinstance(matches, list):
+                    for table in matches:
+                        table_name = table.split(".")[-1]
+                        script_tables.append(QlikTable(tableName=table_name))
+            if not script_tables:
+                logger.warning("No script tables found")
+            return script_tables
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the script tables")
+        return script_tables

metadata/ingestion/source/dashboard/qlikcloud/constants.py

@@ -64,3 +64,9 @@ GET_LOADMODEL_LAYOUT = {
     "id": 5,
     "jsonrpc": "2.0",
 }
+GET_SCRIPT = {
+    "handle": 3,
+    "method": "GetScript",
+    "id": 1,
+    "jsonrpc": "2.0",
+}

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -84,3 +84,11 @@ class QlikAppResponse(BaseModel):
 
     apps: Optional[List[QlikApp]] = Field(None, alias="data")
     links: Optional[QlikLinks] = None
+
+
+class QlikScript(BaseModel):
+    qScript: Optional[str] = None
+
+
+class QlikScriptResult(BaseModel):
+    result: Optional[QlikScript] = QlikScript()

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -11,6 +11,7 @@
 """
 Tableau source module
 """
+# pylint: disable=too-many-lines
 import traceback
 from datetime import datetime
 from typing import Any, Iterable, List, Optional, Set
@@ -62,7 +63,10 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import get_column_fqn
+from metadata.ingestion.lineage.sql_lineage import (
+    get_column_fqn,
+    get_table_fqn_from_query_name,
+)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import (
@@ -157,13 +161,13 @@ class TableauSource(DashboardServiceSource):
         return None
 
     @staticmethod
-    def _get_data_models_tags(dataModels: List[DataSource]) -> Set[str]:
+    def _get_data_models_tags(data_models: List[DataSource]) -> Set[str]:
         """
         Get the tags from the data model in the upstreamDatasources
         """
         tags = set()
         try:
-            for data_model in dataModels:
+            for data_model in data_models:
                 # tags seems to be available for upstreamDatasources only, not for dataModels
                 for upstream_source in data_model.upstreamDatasources or []:
                     for tag in upstream_source.tags:
@@ -191,7 +195,7 @@ class TableauSource(DashboardServiceSource):
             _all_tags = tags.union(_data_models_tags)
 
             yield from get_ometa_tag_and_classification(
-                tags=[tag for tag in _all_tags],
+                tags=list(_all_tags),
                 classification_name=TABLEAU_TAG_CATEGORY,
                 tag_description="Tableau Tag",
                 classification_description="Tags associated with tableau entities",
@@ -370,6 +374,7 @@ class TableauSource(DashboardServiceSource):
                     columns.append(child_column.fullyQualifiedName.root)
         return columns
 
+    # pylint: disable=arguments-differ
     def _get_column_lineage(
         self,
         upstream_table: UpstreamTable,
@@ -562,6 +567,7 @@ class TableauSource(DashboardServiceSource):
 
         return datamodel_column_lineage or None
 
+    # pylint: disable=too-many-locals,too-many-nested-blocks
     def _get_datamodel_table_lineage(
         self,
         datamodel: DataSource,
@@ -644,7 +650,8 @@ class TableauSource(DashboardServiceSource):
                                 )
                                 if not from_entities:
                                     logger.debug(
-                                        f"No table entities found for custom SQL lineage. fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
+                                        "No table entities found for custom SQL lineage."
+                                        f"fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
                                     )
                                 for table_entity in from_entities:
                                     yield self._get_add_lineage_request(
@@ -817,8 +824,36 @@ class TableauSource(DashboardServiceSource):
             )
             if table_entity:
                 return [TableAndQuery(table=table_entity)]
+
+            # fallback to fullName if schema name is not appropriate
+            # TODO: We should make this a primary approach for lineage instead
+            # using this strategy as a fallback.
+            if not table.fullName:
+                logger.debug(
+                    "No table entity found for lineage using GraphQL APIs."
+                    f"fqn_search_string={fqn_search_string}, table_name={table_name}"
+                )
+                return None
+            # fullname is in the format of [database].[schema].[table] or [database].[table] or [table]
+            database_query, schema_query, table_query = get_table_fqn_from_query_name(
+                table.fullName.replace("[", "").replace("]", "")
+            )
+            if table_query:
+                fqn_search_string = build_es_fqn_search_string(
+                    database_name=database_query,
+                    schema_name=schema_query,
+                    service_name=db_service_name or "*",
+                    table_name=table_query,
+                )
+                table_entity = self.metadata.search_in_any_service(
+                    entity_type=Table,
+                    fqn_search_string=fqn_search_string,
+                )
+                if table_entity:
+                    return [TableAndQuery(table=table_entity)]
             logger.debug(
-                f"No table entity found for lineage using GraphQL APIs. fqn_search_string={fqn_search_string}, table_name={table_name}"
+                "No table entity found for lineage using GraphQL APIs."
+                f"fqn_search_string={fqn_search_string}, table_name={table_name}"
             )
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -875,7 +910,9 @@ class TableauSource(DashboardServiceSource):
                     )
                     if not from_entities:
                         logger.debug(
-                            f"No table entities found for lineage using SQL Queries. fqn_search_string={fqn_search_string}, table_name={table_name}, query={custom_sql_table.query}"
+                            "No table entities found for lineage using SQL Queries."
+                            f"fqn_search_string={fqn_search_string}, "
+                            f"table_name={table_name}, query={custom_sql_table.query}"
                         )
                     tables_list.extend(
                         [

metadata/ingestion/source/database/databricks/client.py

@@ -14,14 +14,23 @@ Client to interact with databricks apis
 import json
 import traceback
 from datetime import timedelta
-from typing import Iterable, List
+from typing import Iterable, List, Optional, Tuple, Union
 
 import requests
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
 
 from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
     DatabricksConnection,
 )
+from metadata.generated.schema.entity.services.connections.pipeline.databricksPipelineConnection import (
+    DatabricksPipelineConnection,
+)
 from metadata.ingestion.ometa.client import APIError
+from metadata.ingestion.source.database.databricks.queries import (
+    DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB,
+    DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB,
+)
 from metadata.utils.constants import QUERY_WITH_DBT, QUERY_WITH_OM_VERSION
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger
@@ -30,6 +39,8 @@ logger = ingestion_logger()
 API_TIMEOUT = 10
 PAGE_SIZE = 100
 QUERIES_PATH = "/sql/history/queries"
+API_VERSION = "/api/2.0"
+JOB_API_VERSION = "/api/2.1"
 
 
 class DatabricksClientException(Exception):
@@ -43,30 +54,58 @@ class DatabricksClient:
     DatabricksClient creates a Databricks connection based on DatabricksCredentials.
     """
 
-    def __init__(self, config: DatabricksConnection):
+    def __init__(
+        self,
+        config: Union[DatabricksConnection, DatabricksPipelineConnection],
+        engine: Optional[Engine] = None,
+    ):
         self.config = config
         base_url, *_ = self.config.hostPort.split(":")
-        api_version = "/api/2.0"
-        job_api_version = "/api/2.1"
         auth_token = self.config.token.get_secret_value()
-        self.base_url = f"https://{base_url}{api_version}"
+        self.base_url = f"https://{base_url}{API_VERSION}"
         self.base_query_url = f"{self.base_url}{QUERIES_PATH}"
-        self.base_job_url = f"https://{base_url}{job_api_version}/jobs"
+        self.base_job_url = f"https://{base_url}{JOB_API_VERSION}/jobs"
         self.jobs_list_url = f"{self.base_job_url}/list"
         self.jobs_run_list_url = f"{self.base_job_url}/runs/list"
         self.headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
         }
+        self.api_timeout = self.config.connectionTimeout or 120
+        self.job_table_lineage: dict[str, list[dict[str, str]]] = {}
+        self.job_column_lineage: dict[
+            str, dict[Tuple[str, str], list[Tuple[str, str]]]
+        ] = {}
+        self.engine = engine
         self.client = requests
 
     def test_query_api_access(self) -> None:
         res = self.client.get(
-            self.base_query_url, headers=self.headers, timeout=API_TIMEOUT
+            self.base_query_url, headers=self.headers, timeout=self.api_timeout
         )
         if res.status_code != 200:
             raise APIError(res.json)
 
+    def test_lineage_query(self) -> None:
+        try:
+            with self.engine.connect() as connection:
+                test_table_lineage = connection.execute(
+                    text(DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB + " LIMIT 1")
+                )
+                test_column_lineage = connection.execute(
+                    text(DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB + " LIMIT 1")
+                )
+                # Check if queries executed successfully by fetching results
+                table_result = test_table_lineage.fetchone()
+                column_result = test_column_lineage.fetchone()
+                logger.info("Lineage queries executed successfully")
+        except Exception as exc:
+            logger.debug(f"Error testing lineage queries: {traceback.format_exc()}")
+            raise DatabricksClientException(
+                f"Failed to test lineage queries Make sure you have access"
+                "to the tables table_lineage and column_lineage: {exc}"
+            )
+
     def _run_query_paginator(self, data, result, end_time, response):
         while True:
             if response:
@@ -85,7 +124,7 @@ class DatabricksClient:
                     self.base_query_url,
                     data=json.dumps(data),
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
                 yield from response.get("res") or []
 
@@ -117,7 +156,7 @@ class DatabricksClient:
                         self.base_query_url,
                         data=json.dumps(data),
                         headers=self.headers,
-                        timeout=API_TIMEOUT,
+                        timeout=self.api_timeout,
                     ).json()
 
                     result = response.get("res") or []
@@ -145,7 +184,7 @@ class DatabricksClient:
             self.jobs_list_url,
             data=json.dumps(data),
             headers=self.headers,
-            timeout=API_TIMEOUT,
+            timeout=self.api_timeout,
         )
         if response.status_code != 200:
             raise DatabricksClientException(response.text)
@@ -154,6 +193,7 @@ class DatabricksClient:
         """
         Method returns List all the created jobs in a Databricks Workspace
         """
+        self.cache_lineage()
         try:
             iteration_count = 1
             data = {"limit": PAGE_SIZE, "expand_tasks": True, "offset": 0}
@@ -162,7 +202,7 @@ class DatabricksClient:
                 self.jobs_list_url,
                 data=json.dumps(data),
                 headers=self.headers,
-                timeout=API_TIMEOUT,
+                timeout=self.api_timeout,
             ).json()
 
             yield from response.get("jobs") or []
@@ -174,7 +214,7 @@ class DatabricksClient:
                     self.jobs_list_url,
                     data=json.dumps(data),
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
                 iteration_count += 1
                 yield from response.get("jobs") or []
@@ -200,7 +240,7 @@ class DatabricksClient:
                 self.jobs_run_list_url,
                 params=params,
                 headers=self.headers,
-                timeout=API_TIMEOUT,
+                timeout=self.api_timeout,
             ).json()
 
             yield from response.get("runs") or []
@@ -212,7 +252,7 @@ class DatabricksClient:
                     self.jobs_run_list_url,
                     params=params,
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
 
                 yield from response.get("runs") or []
@@ -220,3 +260,98 @@ class DatabricksClient:
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(exc)
+
+    def get_table_lineage(self, job_id: str) -> List[dict[str, str]]:
+        """
+        Method returns table lineage for a job by the specified job_id
+        """
+        try:
+            return self.job_table_lineage.get(str(job_id))
+        except Exception as exc:
+            logger.debug(
+                f"Error getting table lineage for job {job_id} due to {traceback.format_exc()}"
+            )
+            logger.error(exc)
+        return []
+
+    def get_column_lineage(
+        self, job_id: str, TableKey: Tuple[str, str]
+    ) -> List[Tuple[str, str]]:
+        """
+        Method returns column lineage for a job by the specified job_id and table key
+        """
+        try:
+            return self.job_column_lineage.get(str(job_id), {}).get(TableKey)
+        except Exception as exc:
+            logger.debug(
+                f"Error getting column lineage for table {TableKey} due to {traceback.format_exc()}"
+            )
+            logger.error(exc)
+        return []
+
+    def run_lineage_query(self, query: str) -> List[dict]:
+        """
+        Method runs a lineage query and returns the result
+        """
+        try:
+            with self.engine.connect() as connection:
+                result = connection.execute(text(query))
+                return result
+
+        except Exception as exc:
+            logger.debug(f"Error caching table lineage due to {traceback.format_exc()}")
+            logger.error(exc)
+        return []
+
+    def cache_lineage(self):
+        """
+        Method caches table and column lineage for a job by the specified job_id
+        """
+        logger.info(f"Caching table lineage")
+        table_lineage = self.run_lineage_query(DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB)
+        if table_lineage:
+            for row in table_lineage:
+                try:
+                    if row.job_id not in self.job_table_lineage:
+                        self.job_table_lineage[row.job_id] = []
+                    self.job_table_lineage[row.job_id].append(
+                        {
+                            "source_table_full_name": row.source_table_full_name,
+                            "target_table_full_name": row.target_table_full_name,
+                        }
+                    )
+                except Exception as exc:
+                    logger.debug(
+                        f"Error parsing row: {row} due to {traceback.format_exc()}"
+                    )
+                    continue
+
+        # Not every job has column lineage, so we need to check if the job exists in the column_lineage table
+        # we will cache the column lineage for jobs that have column lineage
+        logger.info("Caching column lineage")
+        column_lineage = self.run_lineage_query(DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB)
+        if column_lineage:
+            for row in column_lineage:
+                try:
+                    table_key = (
+                        row.source_table_full_name,
+                        row.target_table_full_name,
+                    )
+                    column_pair = (
+                        row.source_column_name,
+                        row.target_column_name,
+                    )
+
+                    if row.job_id not in self.job_column_lineage:
+                        self.job_column_lineage[row.job_id] = {}
+
+                    if table_key not in self.job_column_lineage[row.job_id]:
+                        self.job_column_lineage[row.job_id][table_key] = []
+
+                    self.job_column_lineage[row.job_id][table_key].append(column_pair)
+
+                except Exception as exc:
+                    logger.debug(
+                        f"Error parsing row: {row} due to {traceback.format_exc()}"
+                    )
+                    continue