openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -466,9 +466,9 @@ class PowerbiSource(DashboardServiceSource):
         for column in table.columns or []:
             try:
                 parsed_column = {
-                    "dataTypeDisplay": column.dataType
-                    if column.dataType
-                    else DataType.UNKNOWN.value,
+                    "dataTypeDisplay": (
+                        column.dataType if column.dataType else DataType.UNKNOWN.value
+                    ),
                     "dataType": ColumnTypeParser.get_column_type(
                         column.dataType if column.dataType else None
                     ),
@@ -489,11 +489,20 @@ class PowerbiSource(DashboardServiceSource):
         datasource_columns = []
         for table in dataset.tables or []:
             try:
+                table_display_name = None
+                if self.service_connection.displayTableNameFromSource:
+                    table_display_name = self.parse_table_name_from_source(table=table)
+                    if table_display_name:
+                        logger.debug(
+                            f"Parsed Table display name: {table_display_name} for table: {table.name}"
+                        )
+                if not table_display_name:
+                    table_display_name = table.name
                 parsed_table = {
                     "dataTypeDisplay": "PowerBI Table",
                     "dataType": DataType.TABLE,
                     "name": table.name,
-                    "displayName": table.name,
+                    "displayName": table_display_name,
                     "description": table.description,
                     "children": [],
                 }
@@ -547,9 +556,11 @@ class PowerbiSource(DashboardServiceSource):
                     data_model_request = CreateDashboardDataModelRequest(
                         name=EntityName(dataset.id),
                         displayName=dataset.name,
-                        description=Markdown(dataset.description)
-                        if dataset.description
-                        else None,
+                        description=(
+                            Markdown(dataset.description)
+                            if dataset.description
+                            else None
+                        ),
                         service=FullyQualifiedEntityName(
                             self.context.get().dashboard_service
                         ),
@@ -615,11 +626,13 @@ class PowerbiSource(DashboardServiceSource):
 
     def create_datamodel_report_lineage(
         self,
+        db_service_prefix: Optional[str],
         dashboard_details: PowerBIReport,
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """
         create the lineage between datamodel and report
         """
+
         try:
             report_fqn = fqn.build(
                 self.metadata,
@@ -683,6 +696,41 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(f"Error to get data_model_column_fqn {exc}")
             logger.debug(traceback.format_exc())
 
+    def parse_table_name_from_source(self, table: PowerBiTable) -> Optional[str]:
+        """
+        Parse the snowflake table name
+        """
+        try:
+            if not isinstance(table.source, list):
+                return None
+            source_expression = table.source[0].expression
+            if not source_expression:
+                logger.debug(f"No source expression found for table: {table.name}")
+                return None
+
+            if "Snowflake.Databases" in source_expression:
+                # snowflake expression
+                table_match = re.search(
+                    r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Table"\]', source_expression
+                )
+                view_match = re.search(
+                    r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="View"\]', source_expression
+                )
+                table = table_match.group(1) if table_match else None
+                view = view_match.group(1) if view_match else None
+                return table if table else view
+
+            # other general expressions
+            table_match = re.findall(r'\[Name="([^"]+)"\]', source_expression)
+            table = None
+            if isinstance(table_match, list):
+                table = table_match[1] if len(table_match) > 1 else None
+            return table
+        except Exception as exc:
+            logger.debug(f"Error to parse display table name: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
     def _parse_snowflake_regex_exp(
         self, match: re.Match, datamodel_entity: DashboardDataModel
     ) -> Optional[str]:
@@ -802,20 +850,60 @@ class PowerbiSource(DashboardServiceSource):
 
     def _get_table_and_datamodel_lineage(
         self,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         table: PowerBiTable,
         datamodel_entity: DashboardDataModel,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels
         """
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         try:
             table_info = self._parse_table_info_from_source_exp(table, datamodel_entity)
+            table_name = table.name or table_info.get("table")
+            schema_name = table_info.get("schema")
+            database_name = table_info.get("database")
+            if (
+                prefix_table_name
+                and table_name
+                and prefix_table_name.lower() != table_name.lower()
+            ):
+                logger.debug(
+                    f"Table {table_name} does not match prefix {prefix_table_name}"
+                )
+                return
+
+            if (
+                prefix_schema_name
+                and schema_name
+                and prefix_schema_name.lower() != schema_name.lower()
+            ):
+                logger.debug(
+                    f"Schema {table_info.get('schema')} does not match prefix {prefix_schema_name}"
+                )
+                return
+
+            if (
+                prefix_database_name
+                and database_name
+                and prefix_database_name.lower() != database_name.lower()
+            ):
+                logger.debug(
+                    f"Database {table_info.get('database')} does not match prefix {prefix_database_name}"
+                )
+                return
+
             fqn_search_string = build_es_fqn_search_string(
-                service_name=db_service_name or "*",
-                table_name=table_info.get("table") or table.name,
-                schema_name=table_info.get("schema") or "*",
-                database_name=table_info.get("database") or "*",
+                service_name=prefix_service_name or "*",
+                table_name=(prefix_table_name or table_name),
+                schema_name=(prefix_schema_name or schema_name),
+                database_name=(prefix_database_name or database_name),
             )
             table_entity = self.metadata.search_in_any_service(
                 entity_type=Table,
@@ -845,12 +933,14 @@ class PowerbiSource(DashboardServiceSource):
 
     def create_table_datamodel_lineage_from_files(
         self,
-        db_service_name: Optional[str],
+        db_service_prefix: Optional[str],
         datamodel_entity: Optional[DashboardDataModel],
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels using pbit files
         """
+        (prefix_service_name, *_) = self.parse_db_service_prefix(db_service_prefix)
+
         try:
             # check if the datamodel_file_mappings is populated or not
             # if not, then populate the datamodel_file_mappings and process the lineage
@@ -871,7 +961,7 @@ class PowerbiSource(DashboardServiceSource):
             for datamodel_schema_file in datamodel_file_list:
                 for table in datamodel_schema_file.tables or []:
                     yield from self._get_table_and_datamodel_lineage(
-                        db_service_name=db_service_name,
+                        db_service_prefix=db_service_prefix,
                         table=table,
                         datamodel_entity=datamodel_entity,
                     )
@@ -881,7 +971,7 @@ class PowerbiSource(DashboardServiceSource):
                     name="DataModel Lineage",
                     error=(
                         "Error to yield datamodel lineage details for DB "
-                        f"service name [{str(db_service_name)}]: {exc}"
+                        f"service name [{prefix_service_name}]: {exc}"
                     ),
                     stackTrace=traceback.format_exc(),
                 )
@@ -1031,17 +1121,20 @@ class PowerbiSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Group,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         We will build the logic to build the logic as below
         tables - datamodel - report - dashboard
         """
+        (prefix_service_name, *_) = self.parse_db_service_prefix(db_service_prefix)
+
         for dashboard in self.filtered_dashboards or []:
             dashboard_details = self.get_dashboard_details(dashboard)
             try:
                 if isinstance(dashboard_details, PowerBIReport):
                     yield from self.create_datamodel_report_lineage(
+                        db_service_prefix=db_service_prefix,
                         dashboard_details=dashboard_details,
                     )
                 if isinstance(dashboard_details, PowerBIDashboard):
@@ -1052,7 +1145,7 @@ class PowerbiSource(DashboardServiceSource):
                 yield Either(
                     left=StackTraceError(
                         name="Dashboard Lineage",
-                        error=f"Error to yield dashboard lineage details for DB service name [{str(db_service_name)}]: {exc}",
+                        error=f"Error to yield dashboard lineage details for DB service name [{str(prefix_service_name)}]: {exc}",
                         stackTrace=traceback.format_exc(),
                     )
                 )
@@ -1082,7 +1175,7 @@ class PowerbiSource(DashboardServiceSource):
                         # 1. datamodel-db_table lineage
                         for table in datamodel.tables or []:
                             yield from self._get_table_and_datamodel_lineage(
-                                db_service_name=db_service_name,
+                                db_service_prefix=db_service_prefix,
                                 table=table,
                                 datamodel_entity=datamodel_entity,
                             )
@@ -1097,7 +1190,7 @@ class PowerbiSource(DashboardServiceSource):
                         # create the lineage between table and datamodel using the pbit files
                         if self.client.file_client:
                             yield from self.create_table_datamodel_lineage_from_files(
-                                db_service_name=db_service_name,
+                                db_service_prefix=db_service_prefix,
                                 datamodel_entity=datamodel_entity,
                             )
                     elif isinstance(datamodel, Dataflow):
@@ -1113,7 +1206,7 @@ class PowerbiSource(DashboardServiceSource):
                 yield Either(
                     left=StackTraceError(
                         name="Datamodel Lineage",
-                        error=f"Error to yield datamodel lineage details for DB service name [{str(db_service_name)}]: {exc}",
+                        error=f"Error to yield datamodel lineage details for DB service name [{str(prefix_service_name)}]: {exc}",
                         stackTrace=traceback.format_exc(),
                     )
                 )

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -12,6 +12,7 @@
 REST Auth & Client for QlikCloud
 """
 import json
+import re
 import traceback
 from typing import Dict, Iterable, List, Optional
 
@@ -23,12 +24,14 @@ from metadata.ingestion.source.dashboard.qlikcloud.constants import (
     APP_LOADMODEL_REQ,
     CREATE_SHEET_SESSION,
     GET_LOADMODEL_LAYOUT,
+    GET_SCRIPT,
     GET_SHEET_LAYOUT,
     OPEN_DOC_REQ,
 )
 from metadata.ingestion.source.dashboard.qlikcloud.models import (
     QlikApp,
     QlikAppResponse,
+    QlikScriptResult,
     QlikSpace,
     QlikSpaceResponse,
 )
@@ -173,12 +176,18 @@ class QlikCloudClient:
             models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True)
             data_models = QlikDataModelResult(**models)
             layout = data_models.result.qLayout
+            parsed_datamodels = []
             if isinstance(layout, list):
                 tables = []
                 for layout in data_models.result.qLayout:
                     tables.extend(layout.value.tables)
-                return tables
-            return layout.tables
+                parsed_datamodels.extend(tables)
+            else:
+                parsed_datamodels.extend(layout.tables)
+            script_tables = self.get_script_tables()
+            if script_tables:
+                parsed_datamodels.extend(script_tables)
+            return parsed_datamodels
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the dashboard datamodels")
@@ -204,3 +213,26 @@ class QlikCloudClient:
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the space list")
+
+    def get_script_tables(self) -> Optional[List[QlikTable]]:
+        """Get script tables from the dashboard script"""
+        script_tables = []
+        try:
+            script_response = self._websocket_send_request(GET_SCRIPT, response=True)
+            script_result = QlikScriptResult(**script_response)
+            if script_result.result.qScript:
+                script_value = script_result.result.qScript
+                matches = re.findall(
+                    r'FROM\s+["\']?([a-zA-Z0-9_.]+)["\']?', script_value, re.IGNORECASE
+                )
+                if isinstance(matches, list):
+                    for table in matches:
+                        table_name = table.split(".")[-1]
+                        script_tables.append(QlikTable(tableName=table_name))
+            if not script_tables:
+                logger.warning("No script tables found")
+            return script_tables
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the script tables")
+        return script_tables

metadata/ingestion/source/dashboard/qlikcloud/constants.py

@@ -64,3 +64,9 @@ GET_LOADMODEL_LAYOUT = {
     "id": 5,
     "jsonrpc": "2.0",
 }
+GET_SCRIPT = {
+    "handle": 3,
+    "method": "GetScript",
+    "id": 1,
+    "jsonrpc": "2.0",
+}

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -240,20 +240,35 @@ class QlikcloudSource(QliksenseSource):
         return None
 
     def yield_dashboard_lineage_details(
-        self,
-        dashboard_details: QlikApp,
-        db_service_name: Optional[str] = None,
+        self, dashboard_details: QlikApp, db_service_prefix: Optional[str] = None
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(datamodel_id=datamodel.id)
                 if data_model_entity:
+                    if (
+                        prefix_table_name
+                        and data_model_entity.displayName
+                        and prefix_table_name.lower()
+                        != data_model_entity.displayName.lower()
+                    ):
+                        logger.debug(
+                            f"Table {data_model_entity.displayName} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=None,
-                        schema_name=None,
-                        service_name=db_service_name or "*",
-                        table_name=data_model_entity.displayName,
+                        database_name=prefix_database_name,
+                        schema_name=prefix_schema_name,
+                        service_name=prefix_service_name or "*",
+                        table_name=prefix_table_name or data_model_entity.displayName,
                     )
                     om_table = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -275,7 +290,7 @@ class QlikcloudSource(QliksenseSource):
                         name=f"{dashboard_details.name} Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {err}"
+                            f"service name [{prefix_service_name}]: {err}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -84,3 +84,11 @@ class QlikAppResponse(BaseModel):
 
     apps: Optional[List[QlikApp]] = Field(None, alias="data")
     links: Optional[QlikLinks] = None
+
+
+class QlikScript(BaseModel):
+    qScript: Optional[str] = None
+
+
+class QlikScriptResult(BaseModel):
+    result: Optional[QlikScript] = QlikScript()

metadata/ingestion/source/dashboard/qliksense/metadata.py

@@ -136,9 +136,11 @@ class QliksenseSource(DashboardServiceSource):
                 name=EntityName(dashboard_details.qDocId),
                 sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.qDocName,
-                description=Markdown(dashboard_details.qMeta.description)
-                if dashboard_details.qMeta.description
-                else None,
+                description=(
+                    Markdown(dashboard_details.qMeta.description)
+                    if dashboard_details.qMeta.description
+                    else None
+                ),
                 charts=[
                     FullyQualifiedEntityName(
                         fqn.build(
@@ -189,9 +191,11 @@ class QliksenseSource(DashboardServiceSource):
                     right=CreateChartRequest(
                         name=EntityName(chart.qInfo.qId),
                         displayName=chart.qMeta.title,
-                        description=Markdown(chart.qMeta.description)
-                        if chart.qMeta.description
-                        else None,
+                        description=(
+                            Markdown(chart.qMeta.description)
+                            if chart.qMeta.description
+                            else None
+                        ),
                         chartType=ChartType.Other,
                         sourceUrl=SourceUrl(chart_url),
                         service=FullyQualifiedEntityName(
@@ -310,9 +314,15 @@ class QliksenseSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: QlikDashboard,
-        db_service_name: Optional[str] = None,
+        db_service_prefix: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
         for datamodel in self.data_models or []:
             try:
                 data_model_entity = self._get_datamodel(datamodel_id=datamodel.id)
@@ -327,11 +337,42 @@ class QliksenseSource(DashboardServiceSource):
                         database_name = None
                     else:
                         schema_name, database_name = None, None
+
+                    if (
+                        prefix_table_name
+                        and datamodel.tableName
+                        and prefix_table_name.lower() != datamodel.tableName.lower()
+                    ):
+                        logger.debug(
+                            f"Table {datamodel.tableName} does not match prefix {prefix_table_name}"
+                        )
+                        continue
+
+                    if (
+                        prefix_schema_name
+                        and schema_name
+                        and prefix_schema_name.lower() != schema_name.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {schema_name} does not match prefix {prefix_schema_name}"
+                        )
+                        continue
+
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {database_name} does not match prefix {prefix_database_name}"
+                        )
+                        continue
+
                     fqn_search_string = build_es_fqn_search_string(
-                        database_name=database_name,
-                        schema_name=schema_name,
-                        service_name=db_service_name or "*",
-                        table_name=datamodel.tableName,
+                        database_name=prefix_database_name or database_name,
+                        schema_name=prefix_schema_name or schema_name,
+                        service_name=prefix_service_name or "*",
+                        table_name=prefix_table_name or datamodel.tableName,
                     )
                     om_table = self.metadata.search_in_any_service(
                         entity_type=Table,
@@ -353,7 +394,7 @@ class QliksenseSource(DashboardServiceSource):
                         name=f"{dashboard_details.qDocName} Lineage",
                         error=(
                             "Error to yield dashboard lineage details for DB "
-                            f"service name [{db_service_name}]: {err}"
+                            f"service name [{prefix_service_name}]: {err}"
                         ),
                         stackTrace=traceback.format_exc(),
                     )