openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0rc1__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -44,6 +44,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.powerbi.models import (
     Dataset,
@@ -82,6 +83,7 @@ class PowerbiSource(DashboardServiceSource):
             100, self.service_connection.pagination_entity_per_page
         )
         self.workspace_data = []
+        self.datamodel_file_mappings = []
 
     def prepare(self):
         if self.service_connection.useAdminApis:
@@ -92,6 +94,10 @@ class PowerbiSource(DashboardServiceSource):
             self.workspace_data = self.get_filtered_workspaces(groups)
         return super().prepare()
 
+    def close(self):
+        self.metadata.close()
+        self.client.file_client.delete_tmp_files()
+
     def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
         """
         Method to get the workspaces filtered by project filter pattern
@@ -114,16 +120,16 @@ class PowerbiSource(DashboardServiceSource):
         """
         fetch all the group workspace ids
         """
-        groups = self.client.fetch_all_workspaces()
+        groups = self.client.api_client.fetch_all_workspaces()
         for group in groups:
             # add the dashboards to the groups
             group.dashboards.extend(
-                self.client.fetch_all_org_dashboards(group_id=group.id) or []
+                self.client.api_client.fetch_all_org_dashboards(group_id=group.id) or []
             )
             for dashboard in group.dashboards:
                 # add the tiles to the dashboards
                 dashboard.tiles.extend(
-                    self.client.fetch_all_org_tiles(
+                    self.client.api_client.fetch_all_org_tiles(
                         group_id=group.id, dashboard_id=dashboard.id
                     )
                     or []
@@ -131,17 +137,17 @@ class PowerbiSource(DashboardServiceSource):
 
             # add the reports to the groups
             group.reports.extend(
-                self.client.fetch_all_org_reports(group_id=group.id) or []
+                self.client.api_client.fetch_all_org_reports(group_id=group.id) or []
             )
 
             # add the datasets to the groups
             group.datasets.extend(
-                self.client.fetch_all_org_datasets(group_id=group.id) or []
+                self.client.api_client.fetch_all_org_datasets(group_id=group.id) or []
             )
             for dataset in group.datasets:
                 # add the tables to the datasets
                 dataset.tables.extend(
-                    self.client.fetch_dataset_tables(
+                    self.client.api_client.fetch_dataset_tables(
                         group_id=group.id, dataset_id=dataset.id
                     )
                     or []
@@ -153,7 +159,7 @@ class PowerbiSource(DashboardServiceSource):
         fetch all the workspace ids
         """
         groups = []
-        workspaces = self.client.fetch_all_workspaces()
+        workspaces = self.client.api_client.fetch_all_workspaces()
         if workspaces:
             workspace_id_list = [workspace.id for workspace in workspaces]
 
@@ -169,16 +175,16 @@ class PowerbiSource(DashboardServiceSource):
                 logger.info(
                     f"Scanning {count}/{len(workspace_paginated_list)} set of workspaces"
                 )
-                workspace_scan = self.client.initiate_workspace_scan(
+                workspace_scan = self.client.api_client.initiate_workspace_scan(
                     workspace_ids_chunk
                 )
 
                 # Keep polling the scan status endpoint to check if scan is succeeded
-                workspace_scan_status = self.client.wait_for_scan_complete(
+                workspace_scan_status = self.client.api_client.wait_for_scan_complete(
                     scan_id=workspace_scan.id
                 )
                 if workspace_scan_status:
-                    response = self.client.fetch_workspace_scan_result(
+                    response = self.client.api_client.fetch_workspace_scan_result(
                         scan_id=workspace_scan.id
                     )
                     groups.extend(
@@ -196,7 +202,9 @@ class PowerbiSource(DashboardServiceSource):
         return groups or None
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config = WorkflowSource.parse_obj(config_dict)
         connection: PowerBIConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, PowerBIConnection):
@@ -210,7 +218,7 @@ class PowerbiSource(DashboardServiceSource):
         Method to iterate through dashboard lists filter dashboards & yield dashboard details
         """
         for workspace in self.workspace_data:
-            self.context.workspace = workspace
+            self.context.get().workspace = workspace
             for dashboard in self.get_dashboards_list():
                 try:
                     dashboard_details = self.get_dashboard_details(dashboard)
@@ -239,7 +247,10 @@ class PowerbiSource(DashboardServiceSource):
         """
         Get List of all dashboards
         """
-        return self.context.workspace.reports + self.context.workspace.dashboards
+        return (
+            self.context.get().workspace.reports
+            + self.context.get().workspace.dashboards
+        )
 
     def get_dashboard_name(
         self, dashboard: Union[PowerBIDashboard, PowerBIReport]
@@ -320,7 +331,7 @@ class PowerbiSource(DashboardServiceSource):
                 name=dataset.id,
                 displayName=dataset.name,
                 description=dataset.description,
-                service=self.context.dashboard_service,
+                service=self.context.get().dashboard_service,
                 dataModelType=DataModelType.PowerBIDataModel.value,
                 serviceType=DashboardServiceType.PowerBI.value,
                 columns=self._get_column_info(dataset),
@@ -395,7 +406,7 @@ class PowerbiSource(DashboardServiceSource):
                 dashboard_request = CreateDashboardRequest(
                     name=dashboard_details.id,
                     sourceUrl=self._get_dashboard_url(
-                        workspace_id=self.context.workspace.id,
+                        workspace_id=self.context.get().workspace.id,
                         dashboard_id=dashboard_details.id,
                     ),
                     project=self.get_project_name(dashboard_details=dashboard_details),
@@ -405,12 +416,12 @@ class PowerbiSource(DashboardServiceSource):
                         fqn.build(
                             self.metadata,
                             entity_type=Chart,
-                            service_name=self.context.dashboard_service,
+                            service_name=self.context.get().dashboard_service,
                             chart_name=chart,
                         )
-                        for chart in self.context.charts or []
+                        for chart in self.context.get().charts or []
                     ],
-                    service=self.context.dashboard_service,
+                    service=self.context.get().dashboard_service,
                     owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             else:
@@ -418,12 +429,12 @@ class PowerbiSource(DashboardServiceSource):
                     name=dashboard_details.id,
                     dashboardType=DashboardType.Report,
                     sourceUrl=self._get_report_url(
-                        workspace_id=self.context.workspace.id,
+                        workspace_id=self.context.get().workspace.id,
                         dashboard_id=dashboard_details.id,
                     ),
                     project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.name,
-                    service=self.context.dashboard_service,
+                    service=self.context.get().dashboard_service,
                     owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             yield Either(right=dashboard_request)
@@ -522,6 +533,13 @@ class PowerbiSource(DashboardServiceSource):
                         tables=dataset.tables,
                         datamodel_entity=datamodel_entity,
                     )
+
+                    # create the lineage between table and datamodel using the pbit files
+                    if self.client.file_client:
+                        yield from self.create_table_datamodel_lineage_from_files(
+                            db_service_name=db_service_name,
+                            datamodel_entity=datamodel_entity,
+                        )
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
@@ -534,6 +552,90 @@ class PowerbiSource(DashboardServiceSource):
                 )
             )
 
+    def _get_table_and_datamodel_lineage(
+        self,
+        db_service_name: str,
+        table: PowerBiTable,
+        datamodel_entity: DashboardDataModel,
+    ) -> Optional[Iterable[Either[AddLineageRequest]]]:
+        """
+        Method to create lineage between table and datamodels
+        """
+        try:
+            table_fqn = fqn.build(
+                self.metadata,
+                entity_type=Table,
+                service_name=db_service_name,
+                database_name=None,
+                schema_name=None,
+                table_name=table.name,
+            )
+            table_entity = self.metadata.get_by_name(
+                entity=Table,
+                fqn=table_fqn,
+            )
+
+            if table_entity and datamodel_entity:
+                return self._get_add_lineage_request(
+                    to_entity=datamodel_entity, from_entity=table_entity
+                )
+        except Exception as exc:  # pylint: disable=broad-except
+            return Either(
+                left=StackTraceError(
+                    name="DataModel Lineage for pbit files",
+                    error=(
+                        "Error to yield datamodel lineage details using pbit files for"
+                        f"datamodel [{datamodel_entity.name}]: {exc}"
+                    ),
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+        return None
+
+    def create_table_datamodel_lineage_from_files(
+        self,
+        db_service_name: str,
+        datamodel_entity: Optional[DashboardDataModel],
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Method to create lineage between table and datamodels using pbit files
+        """
+        try:
+            # check if the datamodel_file_mappings is populated or not
+            # if not, then populate the datamodel_file_mappings and process the lineage
+            if not self.datamodel_file_mappings:
+                self.datamodel_file_mappings = (
+                    self.client.file_client.get_data_model_schema_mappings()
+                )
+
+            # search which file contains the datamodel and for the given datamodel_entity
+            datamodel_file_list = []
+            for datamodel_schema in self.datamodel_file_mappings or []:
+                for connections in (
+                    datamodel_schema.connectionFile.RemoteArtifacts or []
+                ):
+                    if connections.DatasetId == model_str(datamodel_entity.name):
+                        datamodel_file_list.append(datamodel_schema)
+
+            for datamodel_schema_file in datamodel_file_list:
+                for table in datamodel_schema_file.tables or []:
+                    yield self._get_table_and_datamodel_lineage(
+                        db_service_name=db_service_name,
+                        table=table,
+                        datamodel_entity=datamodel_entity,
+                    )
+        except Exception as exc:  # pylint: disable=broad-except
+            yield Either(
+                left=StackTraceError(
+                    name="DataModel Lineage",
+                    error=(
+                        "Error to yield datamodel lineage details for DB "
+                        f"service name [{db_service_name}]: {exc}"
+                    ),
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
     def create_table_datamodel_lineage(
         self,
         db_service_name: str,
@@ -542,35 +644,11 @@ class PowerbiSource(DashboardServiceSource):
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """Method to create lineage between table and datamodels"""
         for table in tables or []:
-            try:
-                table_fqn = fqn.build(
-                    self.metadata,
-                    entity_type=Table,
-                    service_name=db_service_name,
-                    database_name=None,
-                    schema_name=None,
-                    table_name=table.name,
-                )
-                table_entity = self.metadata.get_by_name(
-                    entity=Table,
-                    fqn=table_fqn,
-                )
-
-                if table_entity and datamodel_entity:
-                    yield self._get_add_lineage_request(
-                        to_entity=datamodel_entity, from_entity=table_entity
-                    )
-            except Exception as exc:  # pylint: disable=broad-except
-                yield Either(
-                    left=StackTraceError(
-                        name="DataModel Lineage",
-                        error=(
-                            "Error to yield datamodel lineage details for DB "
-                            f"service name [{db_service_name}]: {exc}"
-                        ),
-                        stackTrace=traceback.format_exc(),
-                    )
-                )
+            yield self._get_table_and_datamodel_lineage(
+                db_service_name=db_service_name,
+                table=table,
+                datamodel_entity=datamodel_entity,
+            )
 
     def yield_dashboard_lineage_details(
         self,
@@ -630,10 +708,10 @@ class PowerbiSource(DashboardServiceSource):
                             chartType=ChartType.Other.value,
                             sourceUrl=self._get_chart_url(
                                 report_id=chart.reportId,
-                                workspace_id=self.context.workspace.id,
+                                workspace_id=self.context.get().workspace.id,
                                 dashboard_id=dashboard_details.id,
                             ),
-                            service=self.context.dashboard_service,
+                            service=self.context.get().dashboard_service,
                         )
                     )
                 except Exception as exc:
@@ -705,7 +783,7 @@ class PowerbiSource(DashboardServiceSource):
         Get the project / workspace / folder / collection name of the dashboard
         """
         try:
-            return str(self.context.workspace.name)
+            return str(self.context.get().workspace.name)
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -192,3 +192,29 @@ class PowerBiToken(BaseModel):
 
     expires_in: Optional[int]
     access_token: Optional[str]
+
+
+class RemoteArtifacts(BaseModel):
+    """
+    PowerBI RemoteArtifacts Model
+    """
+
+    DatasetId: str
+    ReportId: str
+
+
+class ConnectionFile(BaseModel):
+    """
+    PowerBi Connection File Model
+    """
+
+    RemoteArtifacts: Optional[List[RemoteArtifacts]]
+
+
+class DataModelSchema(BaseModel):
+    """
+    PowerBi Data Model Schema Model
+    """
+
+    tables: Optional[List[PowerBiTable]]
+    connectionFile: Optional[ConnectionFile]

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -0,0 +1,167 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+REST Auth & Client for QlikCloud
+"""
+import json
+import traceback
+from typing import Dict, List, Optional
+
+from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
+    QlikCloudConnection,
+)
+from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.ingestion.source.dashboard.qlikcloud.constants import (
+    APP_LOADMODEL_REQ,
+    CREATE_SHEET_SESSION,
+    GET_LOADMODEL_LAYOUT,
+    GET_SHEET_LAYOUT,
+    OPEN_DOC_REQ,
+)
+from metadata.ingestion.source.dashboard.qlikcloud.models import QlikApp, QlikAppList
+from metadata.ingestion.source.dashboard.qliksense.models import (
+    QlikDataModelResult,
+    QlikSheet,
+    QlikSheetResult,
+    QlikTable,
+)
+from metadata.utils.constants import AUTHORIZATION_HEADER
+from metadata.utils.helpers import clean_uri
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+API_VERSION = "api"
+
+
+class QlikCloudClient:
+    """
+    Client Handling API communication with QlikCloud
+    """
+
+    def __init__(
+        self,
+        config: QlikCloudConnection,
+    ):
+        self.config = config
+        self.socket_connection = None
+        self.config.token = self.config.token.get_secret_value()
+
+        client_config: ClientConfig = ClientConfig(
+            base_url=self.config.hostPort,
+            api_version=API_VERSION,
+            auth_header=AUTHORIZATION_HEADER,
+            auth_token=lambda: (self.config.token, 0),
+        )
+        self.client = REST(client_config)
+
+    def connect_websocket(self, dashboard_id: str = None) -> None:
+        """
+        Method to initialise websocket connection
+        """
+        # pylint: disable=import-outside-toplevel
+        import ssl
+
+        from websocket import create_connection
+
+        if self.socket_connection:
+            self.socket_connection.close()
+        self.socket_connection = create_connection(
+            f"wss://{clean_uri(self.config.hostPort.host)}/app/{dashboard_id or ''}",
+            sslopt={"cert_reqs": ssl.CERT_NONE},
+            header={"Authorization": f"Bearer {self.config.token}"},
+        )
+        self.socket_connection.recv()
+
+    def close_websocket(self) -> None:
+        if self.socket_connection:
+            self.socket_connection.close()
+
+    def _websocket_send_request(
+        self, request: dict, response: bool = False
+    ) -> Optional[Dict]:
+        """
+        Method to send request to websocket
+
+        request: data required to be sent to websocket
+        response: is json response required?
+        """
+        self.socket_connection.send(json.dumps(request))
+        resp = self.socket_connection.recv()
+        if response:
+            return json.loads(resp)
+        return None
+
+    def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]:
+        """
+        Get dahsboard chart list
+        """
+        try:
+            self.connect_websocket(dashboard_id)
+            OPEN_DOC_REQ.update({"params": [dashboard_id]})
+            self._websocket_send_request(OPEN_DOC_REQ)
+            self._websocket_send_request(CREATE_SHEET_SESSION)
+            sheets = self._websocket_send_request(GET_SHEET_LAYOUT, response=True)
+            data = QlikSheetResult(**sheets)
+            return data.result.qLayout.qAppObjectList.qItems
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard charts")
+        return []
+
+    def get_dashboards_list(self) -> List[QlikAppList]:
+        """
+        Get List of all apps
+        """
+        try:
+            resp_apps = self.client.get("/v1/items?resourceType=app")
+            if resp_apps:
+                app_list = QlikAppList(apps=resp_apps.get("data", []))
+                return app_list.apps
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the app list")
+        return []
+
+    def get_dashboard_details(self, dashboard_id: str) -> Optional[QlikApp]:
+        """
+        Get App Details
+        """
+        if not dashboard_id:
+            return None  # don't call api if dashboard_id is None
+        try:
+            resp_dashboard = self.client.get(f"/v1/apps/{dashboard_id}")
+            if resp_dashboard:
+                return QlikApp(**resp_dashboard.get("attributes"))
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
+        return None
+
+    def get_dashboard_models(self) -> List[QlikTable]:
+        """
+        Get dahsboard data models
+        """
+        try:
+            self._websocket_send_request(APP_LOADMODEL_REQ)
+            models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True)
+            data_models = QlikDataModelResult(**models)
+            layout = data_models.result.qLayout
+            if isinstance(layout, list):
+                tables = []
+                for layout in data_models.result.qLayout:
+                    tables.extend(layout.value.tables)
+                return tables
+            return layout.tables
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard datamodels")
+        return []

metadata/ingestion/source/dashboard/qlikcloud/connection.py

@@ -0,0 +1,56 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
+    QlikCloudConnection,
+)
+from metadata.ingestion.connections.test_connections import test_connection_steps
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient
+
+
+def get_connection(connection: QlikCloudConnection) -> QlikCloudClient:
+    """
+    Create connection
+    """
+    return QlikCloudClient(connection)
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    client: QlikCloudClient,
+    service_connection: QlikCloudConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+) -> None:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+
+    def custom_executor():
+        return client.get_dashboards_list()
+
+    test_fn = {"GetDashboards": custom_executor}
+
+    test_connection_steps(
+        metadata=metadata,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
+        automation_workflow=automation_workflow,
+    )

metadata/ingestion/source/dashboard/qlikcloud/constants.py

@@ -0,0 +1,66 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+QlikCloud Constants
+"""
+
+OPEN_DOC_REQ = {
+    "method": "OpenDoc",
+    "handle": -1,
+    "outKey": -1,
+    "id": 1,
+}
+CREATE_SHEET_SESSION = {
+    "method": "CreateSessionObject",
+    "handle": 1,
+    "params": [
+        {
+            "qInfo": {"qType": "SheetList"},
+            "qAppObjectListDef": {
+                "qType": "sheet",
+                "qData": {
+                    "title": "/qMetaDef/title",
+                    "description": "/qMetaDef/description",
+                    "thumbnail": "/thumbnail",
+                    "cells": "/cells",
+                    "rank": "/rank",
+                    "columns": "/columns",
+                    "rows": "/rows",
+                },
+            },
+        }
+    ],
+    "outKey": -1,
+    "id": 2,
+}
+GET_SHEET_LAYOUT = {
+    "method": "GetLayout",
+    "handle": 2,
+    "params": [],
+    "outKey": -1,
+    "id": 3,
+}
+APP_LOADMODEL_REQ = {
+    "delta": True,
+    "handle": 1,
+    "method": "GetObject",
+    "params": ["LoadModel"],
+    "id": 4,
+    "jsonrpc": "2.0",
+}
+GET_LOADMODEL_LAYOUT = {
+    "delta": True,
+    "handle": 3,
+    "method": "GetLayout",
+    "params": [],
+    "id": 5,
+    "jsonrpc": "2.0",
+}