openmetadata-ingestion 1.5.13.2__py3-none-any.whl → 1.6.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py

@@ -17,31 +17,29 @@ from typing import Optional
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
-from metadata.utils.constants import THREE_MIN
+from metadata.ingestion.source.dashboard.microstrategy.client import MicroStrategyClient
 
 
-def get_connection(connection: MstrConnection) -> MSTRClient:
+def get_connection(connection: MicroStrategyConnection) -> MicroStrategyClient:
     """
     Create connection
     """
-    return MSTRClient(connection)
+    return MicroStrategyClient(connection)
 
 
 def test_connection(
     metadata: OpenMetadata,
-    client: MSTRClient,
-    service_connection: MstrConnection,
+    client: MicroStrategyClient,
+    service_connection: MicroStrategyConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
 ) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
@@ -49,7 +47,8 @@ def test_connection(
     """
 
     test_fn = {
-        "GetProjects": client.list_projects,
+        "GetToken": client.get_auth_params,
+        "GetProjects": client.get_projects_list,
     }
 
     return test_connection_steps(
@@ -57,5 +56,4 @@ def test_connection(
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        timeout_seconds=timeout_seconds,
     )

metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py

@@ -8,7 +8,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-"""Mstr source module"""
+"""MicroStrategy source module"""
 import traceback
 from typing import Iterable, List, Optional
 
@@ -16,8 +16,8 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
@@ -34,7 +34,7 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
-from metadata.ingestion.source.dashboard.mstr.models import (
+from metadata.ingestion.source.dashboard.microstrategy.models import (
     MstrDashboard,
     MstrDashboardDetails,
     MstrPage,
@@ -47,9 +47,9 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 
 
-class MstrSource(DashboardServiceSource):
+class MicrostrategySource(DashboardServiceSource):
     """
-    MSTR Source Class
+    Microstrategy Source Class
     """
 
     @classmethod
@@ -60,10 +60,10 @@ class MstrSource(DashboardServiceSource):
         pipeline_name: Optional[str] = None,
     ):
         config = WorkflowSource.model_validate(config_dict)
-        connection: MstrConnection = config.serviceConnection.root.config
-        if not isinstance(connection, MstrConnection):
+        connection: MicroStrategyConnection = config.serviceConnection.root.config
+        if not isinstance(connection, MicroStrategyConnection):
             raise InvalidSourceException(
-                f"Expected MstrConnection, but got {connection}"
+                f"Expected MicroStrategyConnection, but got {connection}"
             )
         return cls(config, metadata)
 
@@ -75,14 +75,18 @@ class MstrSource(DashboardServiceSource):
 
         if self.client.is_project_name():
             project = self.client.get_project_by_name()
-            dashboards.extend(self.client.get_dashboards_list(project.id, project.name))
-
-        if not self.client.is_project_name():
-            for project in self.client.get_projects_list():
+            if project:
                 dashboards.extend(
                     self.client.get_dashboards_list(project.id, project.name)
                 )
 
+        if not self.client.is_project_name():
+            for project in self.client.get_projects_list():
+                if project:
+                    dashboards.extend(
+                        self.client.get_dashboards_list(project.id, project.name)
+                    )
+
         return dashboards
 
     def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
@@ -121,7 +125,7 @@ class MstrSource(DashboardServiceSource):
         if dashboard_details:
             try:
                 dashboard_url = (
-                    f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
+                    f"{clean_uri(str(self.service_connection.hostPort))}/MicroStrategyLibrary/app/"
                     f"{dashboard_details.projectId}/{dashboard_details.id}"
                 )
                 dashboard_request = CreateDashboardRequest(

metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py

@@ -9,7 +9,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """
-MSTR Models
+MicroStrategy Models
 """
 from datetime import datetime
 from typing import Any, List, Optional

metadata/ingestion/source/dashboard/microstrategy/service_spec.py

@@ -0,0 +1,6 @@
+from metadata.ingestion.source.dashboard.microstrategy.metadata import (
+    MicrostrategySource,
+)
+from metadata.utils.service_spec.default import DefaultDatabaseSpec
+
+ServiceSpec = DefaultDatabaseSpec(metadata_source_class=MicrostrategySource)

metadata/ingestion/source/dashboard/mode/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.mode.metadata import ModeSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=ModeSource)

metadata/ingestion/source/dashboard/powerbi/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.powerbi.metadata import PowerbiSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=PowerbiSource)

metadata/ingestion/source/dashboard/qlikcloud/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.qlikcloud.metadata import QlikcloudSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=QlikcloudSource)

metadata/ingestion/source/dashboard/qliksense/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.qliksense.metadata import QliksenseSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=QliksenseSource)

metadata/ingestion/source/dashboard/quicksight/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.quicksight.metadata import QuicksightSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=QuicksightSource)

metadata/ingestion/source/dashboard/redash/service_spec.py

@@ -0,0 +1,4 @@
+from metadata.ingestion.source.dashboard.redash.metadata import RedashSource
+from metadata.utils.service_spec import BaseSpec
+
+ServiceSpec = BaseSpec(metadata_source_class=RedashSource)

metadata/ingestion/source/dashboard/sigma/client.py

@@ -0,0 +1,186 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+REST Auth & Client for Sigma
+"""
+
+import traceback
+from base64 import b64encode
+from typing import List, Optional, Tuple
+
+from metadata.generated.schema.entity.services.connections.dashboard.sigmaConnection import (
+    SigmaConnection,
+)
+from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.ingestion.source.dashboard.sigma.models import (
+    AuthToken,
+    EdgeSourceResponse,
+    Elements,
+    ElementsResponse,
+    NodeDetails,
+    OwnerDetails,
+    Workbook,
+    WorkbookDetails,
+    WorkBookPageResponse,
+    WorkBookResponseDetails,
+)
+from metadata.utils.constants import AUTHORIZATION_HEADER, UTF_8
+from metadata.utils.helpers import clean_uri
+from metadata.utils.logger import utils_logger
+
+logger = utils_logger()
+
+HEADERS = {
+    "accept": "application/json",
+    "Content-type": "application/x-www-form-urlencoded",
+}
+
+TOKEN_PAYLOAD = {"grant_type": "client_credentials"}
+
+
+class SigmaApiClient:
+    """
+    REST Auth & Client for Sigma
+    """
+
+    client: REST
+
+    def __init__(self, config: SigmaConnection):
+        self.config = config
+        token_api_key = str(
+            b64encode(
+                f"{self.config.clientId}:{self.config.clientSecret.get_secret_value()}".encode(
+                    UTF_8
+                )
+            ).decode(UTF_8)
+        )
+
+        token_config = ClientConfig(
+            base_url=clean_uri(config.hostPort),
+            api_version=config.apiVersion,
+            auth_header=AUTHORIZATION_HEADER,
+            extra_headers=HEADERS,
+            auth_token=lambda: (token_api_key, 0),
+            auth_token_mode="Basic",
+        )
+
+        self.token_client = REST(token_config)
+
+        client_config = ClientConfig(
+            base_url=clean_uri(config.hostPort),
+            api_version=config.apiVersion,
+            auth_token=self.get_auth_token,
+            auth_header=AUTHORIZATION_HEADER,
+        )
+
+        self.client = REST(client_config)
+
+    def get_auth_token(self) -> Tuple[str, int]:
+        """
+        generate auth token
+         Returns:
+            Tuple[str, int]: A tuple containing the access_token (str) and expires_in (int)
+        """
+        result = AuthToken.model_validate(
+            self.token_client.post("/auth/token", data=TOKEN_PAYLOAD)
+        )
+        return result.access_token, result.expires_in
+
+    def get_dashboards(self) -> Optional[List[Workbook]]:
+        """
+        method to fetch dashboards from api
+        """
+        result = WorkBookResponseDetails.model_validate(self.client.get("/workbooks"))
+        if result:
+            return result.entries
+
+    def get_dashboard_detail(self, workbook_id: str) -> Optional[WorkbookDetails]:
+        """
+        method to fetch dashboard details from api
+        """
+        try:
+            result = WorkbookDetails.model_validate(
+                self.client.get(f"/workbooks/{workbook_id}")
+            )
+            if result:
+                return result
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(
+                f"Error fetching Dashboard details for for workbook {workbook_id}: {exc}"
+            )
+        return None
+
+    def get_owner_detail(self, owner_id: str) -> Optional[OwnerDetails]:
+        """
+        method to fetch dashboard owner details from api
+        """
+        try:
+            result = OwnerDetails.model_validate(
+                self.client.get(f"/members/{owner_id}")
+            )
+            if result:
+                return result
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch owner details for owner {owner_id}: {exc}")
+        return None
+
+    def get_chart_details(self, workbook_id: str) -> Optional[List[Elements]]:
+        """
+        method to fetch dashboards chart details from api
+        """
+        try:
+            elements_list = []
+            pages = WorkBookPageResponse.model_validate(
+                self.client.get(f"/workbooks/{workbook_id}/pages")
+            )
+            for page in pages.entries:
+                elements = ElementsResponse.model_validate(
+                    self.client.get(
+                        f"/workbooks/{workbook_id}/pages/{page.pageId}/elements"
+                    )
+                )
+                elements_list.extend(elements.entries or [])
+            return elements_list
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to fetch chart details for workbook {workbook_id}: {exc}"
+            )
+        return None
+
+    def get_lineage_details(
+        self, workbook_id: str, element_id: str
+    ) -> Optional[List[NodeDetails]]:
+        """
+        method to fetch dashboards lineage details from api
+        """
+        try:
+            source_nodes = []
+            edges_response = EdgeSourceResponse.model_validate(
+                self.client.get(
+                    f"/workbooks/{workbook_id}/lineage/elements/{element_id}"
+                )
+            )
+            for node in edges_response.edges:
+                if node.node_id:
+                    node_details = NodeDetails.model_validate(
+                        self.client.get(f"/files/{node.node_id}")
+                    )
+                    source_nodes.append(node_details)
+            return source_nodes
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to fetch lineage details for workbook {workbook_id}: {exc}"
+            )
+        return None

metadata/ingestion/source/dashboard/sigma/connection.py

@@ -0,0 +1,67 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler
+"""
+
+from typing import Optional
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.dashboard.sigmaConnection import (
+    SigmaConnection,
+)
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
+from metadata.ingestion.connections.test_connections import (
+    SourceConnectionException,
+    test_connection_steps,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.sigma.client import SigmaApiClient
+from metadata.utils.constants import THREE_MIN
+
+
+def get_connection(connection: SigmaConnection) -> SigmaApiClient:
+    """
+    Create connection
+    """
+    try:
+        return SigmaApiClient(connection)
+    except Exception as exc:
+        msg = f"Unknown error connecting with {connection}: {exc}."
+        raise SourceConnectionException(msg) from exc
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    client: SigmaApiClient,
+    service_connection: SigmaConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+
+    test_fn = {"GetToken": client.get_auth_token, "GetWorkbooks": client.get_dashboards}
+
+    return test_connection_steps(
+        metadata=metadata,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
+        automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
+    )