openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/mstr/client.py

@@ -0,0 +1,208 @@
+#  Copyright 2023 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+REST Auth & Client for Mstr
+"""
+import traceback
+from typing import List, Optional
+
+import requests
+from mstr.requests import MSTRRESTSession
+
+from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
+    MstrConnection,
+)
+from metadata.ingestion.connections.test_connections import SourceConnectionException
+from metadata.ingestion.source.dashboard.mstr.models import (
+    MstrDashboard,
+    MstrDashboardDetails,
+    MstrDashboardList,
+    MstrProject,
+    MstrProjectList,
+    MstrSearchResult,
+    MstrSearchResultList,
+)
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+API_VERSION = "MicroStrategyLibrary/api"
+
+
+class MSTRClient:
+    """
+    Client Handling API communication with Metabase
+    """
+
+    def _get_base_url(self, path=None):
+        if not path:
+            return f"{self.config.hostPort}/{API_VERSION}/"
+        return f"{self.config.hostPort}/{API_VERSION}/{path}"
+
+    def _get_mstr_session(self) -> MSTRRESTSession:
+        try:
+            session = MSTRRESTSession(base_url=self._get_base_url())
+            session.login(
+                username=self.config.username,
+                password=self.config.password.get_secret_value(),
+            )
+            return session
+
+        except KeyError as exe:
+            msg = "Failed to fetch mstr session, please validate credentials"
+            raise SourceConnectionException(msg) from exe
+
+        except Exception as exc:
+            msg = f"Unknown error in connection: {exc}."
+            raise SourceConnectionException(msg) from exc
+
+    def __init__(
+        self,
+        config: MstrConnection,
+    ):
+        self.config = config
+        self.session = self._get_mstr_session()
+
+    def is_project_name(self) -> bool:
+        return bool(self.config.projectName)
+
+    def get_projects_list(self) -> List[MstrProject]:
+        """
+        Get List of all projects
+        """
+        try:
+            resp_projects = self.session.get(
+                url=self._get_base_url("projects"), params={"include_auth": True}
+            )
+
+            if not resp_projects.ok:
+                raise requests.ConnectionError()
+
+            project_list = MstrProjectList(projects=resp_projects.json())
+            return project_list.projects
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the project list due to [{exc}]")
+
+        return []
+
+    def get_project_by_name(self) -> Optional[MstrProject]:
+        """
+        Get Project By Name
+        """
+        try:
+            resp_projects = self.session.get(
+                url=self._get_base_url(f"projects/{self.config.projectName}"),
+                params={"include_auth": True},
+            )
+
+            if not resp_projects.ok:
+                raise requests.ConnectionError()
+
+            project = MstrProject(**resp_projects.json())
+            return project
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the project list")
+
+        return None
+
+    def get_search_results_list(
+        self, project_id, object_type
+    ) -> List[MstrSearchResult]:
+        """
+        Get Search Results
+        """
+        try:
+            resp_results = self.session.get(
+                url=self._get_base_url("searches/results"),
+                params={
+                    "include_auth": True,
+                    "project_id": project_id,
+                    "type": object_type,
+                    "getAncestors": False,
+                    "offset": 0,
+                    "limit": -1,
+                    "certifiedStatus": "ALL",
+                    "isCrossCluster": False,
+                    "result.hidden": False,
+                },
+            )
+
+            if not resp_results.ok:
+                raise requests.ConnectionError()
+
+            results = []
+            for resp_result in resp_results.json()["result"]:
+                results.append(resp_result)
+
+            results_list = MstrSearchResultList(results=results)
+            return results_list.results
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the Search Result list")
+
+        return []
+
+    def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]:
+        """
+        Get Dashboard
+        """
+        try:
+            results = self.get_search_results_list(
+                project_id=project_id, object_type=55
+            )
+
+            dashboards = []
+            for result in results:
+                dashboards.append(
+                    MstrDashboard(projectName=project_name, **result.dict())
+                )
+
+            dashboards_list = MstrDashboardList(dashboards=dashboards)
+            return dashboards_list.dashboards
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard list")
+
+        return []
+
+    def get_dashboard_details(
+        self, project_id, project_name, dashboard_id
+    ) -> Optional[MstrDashboardDetails]:
+        """
+        Get Dashboard Details
+        """
+        try:
+            resp_dashboard = self.session.get(
+                url=self._get_base_url(f"v2/dossiers/{dashboard_id}/definition"),
+                params={
+                    "include_auth": True,
+                },
+                headers={"X-MSTR-ProjectID": project_id},
+            )
+
+            if not resp_dashboard.ok:
+                raise requests.ConnectionError()
+
+            return MstrDashboardDetails(
+                projectId=project_id, projectName=project_name, **resp_dashboard.json()
+            )
+
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
+
+        return None

metadata/ingestion/source/dashboard/mstr/connection.py

@@ -0,0 +1,53 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
+    MstrConnection,
+)
+from metadata.ingestion.connections.test_connections import test_connection_steps
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
+
+
+def get_connection(connection: MstrConnection) -> MSTRClient:
+    """
+    Create connection
+    """
+    return MSTRClient(connection)
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    client: MSTRClient,
+    service_connection: MstrConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+) -> None:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+
+    test_fn = {"GetProjects": client.get_projects_list}
+
+    test_connection_steps(
+        metadata=metadata,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
+        automation_workflow=automation_workflow,
+    )

metadata/ingestion/source/dashboard/mstr/metadata.py

@@ -0,0 +1,182 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Mstr source module"""
+import traceback
+from typing import Iterable, List, Optional
+
+from metadata.generated.schema.api.data.createChart import CreateChartRequest
+from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.chart import Chart
+from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
+    MstrConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.source import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.mstr.models import (
+    MstrDashboard,
+    MstrDashboardDetails,
+    MstrPage,
+)
+from metadata.utils import fqn
+from metadata.utils.filters import filter_by_chart
+from metadata.utils.helpers import clean_uri, get_standard_chart_type
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class MstrSource(DashboardServiceSource):
+    """
+    MSTR Source Class
+    """
+
+    @classmethod
+    def create(cls, config_dict: dict, metadata: OpenMetadata):
+        config = WorkflowSource.parse_obj(config_dict)
+        connection: MstrConnection = config.serviceConnection.__root__.config
+        if not isinstance(connection, MstrConnection):
+            raise InvalidSourceException(
+                f"Expected MstrConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def get_dashboards_list(self) -> Optional[List[MstrDashboard]]:
+        """
+        Get List of all dashboards
+        """
+        dashboards = []
+
+        if self.client.is_project_name():
+            project = self.client.get_project_by_name()
+            dashboards.extend(self.client.get_dashboards_list(project.id, project.name))
+
+        if not self.client.is_project_name():
+            for project in self.client.get_projects_list():
+                dashboards.extend(
+                    self.client.get_dashboards_list(project.id, project.name)
+                )
+
+        return dashboards
+
+    def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
+        """
+        Get Dashboard Name
+        """
+        return dashboard.name
+
+    def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetails:
+        """
+        Get Dashboard Details
+        """
+        dashboard_details = self.client.get_dashboard_details(
+            dashboard.projectId, dashboard.projectName, dashboard.id
+        )
+        return dashboard_details
+
+    def yield_dashboard(
+        self, dashboard_details: MstrDashboardDetails
+    ) -> Iterable[Either[CreateDashboardRequest]]:
+        """
+        Method to Get Dashboard Entity
+        """
+        try:
+            dashboard_url = (
+                f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
+                f"{dashboard_details.projectId}/{dashboard_details.id}"
+            )
+            dashboard_request = CreateDashboardRequest(
+                name=dashboard_details.id,
+                displayName=dashboard_details.name,
+                sourceUrl=dashboard_url,
+                project=dashboard_details.projectName,
+                charts=[
+                    fqn.build(
+                        self.metadata,
+                        entity_type=Chart,
+                        service_name=self.context.dashboard_service,
+                        chart_name=chart,
+                    )
+                    for chart in self.context.charts or []
+                ],
+                service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+            )
+            yield Either(right=dashboard_request)
+            self.register_record(dashboard_request=dashboard_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=dashboard_details.id,
+                    error=f"Error yielding dashboard for {dashboard_details}: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def yield_dashboard_lineage_details(
+        self, dashboard_details: MstrDashboardDetails, db_service_name: str
+    ) -> Optional[Iterable[AddLineageRequest]]:
+        """Not Implemented"""
+
+    def yield_dashboard_chart(
+        self, dashboard_details: MstrDashboardDetails
+    ) -> Optional[Iterable[CreateChartRequest]]:
+        """Get chart method
+
+        Args:
+            dashboard_details:
+        Returns:
+            Iterable[CreateChartRequest]
+        """
+        try:
+            for chapter in dashboard_details.chapters:
+                for page in chapter.pages:
+                    yield from self._yield_chart_from_visualization(page)
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error creating dashboard: {exc}")
+
+    def _yield_chart_from_visualization(
+        self, page: MstrPage
+    ) -> Iterable[Either[CreateChartRequest]]:
+        for chart in page.visualizations:
+            try:
+                if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
+                    self.status.filter(chart.name, "Chart Pattern not allowed")
+                    continue
+
+                yield Either(
+                    right=CreateChartRequest(
+                        name=f"{page.key}{chart.key}",
+                        displayName=chart.name,
+                        chartType=get_standard_chart_type(
+                            chart.visualizationType
+                        ).value,
+                        service=self.context.dashboard_service,
+                    )
+                )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name="Chart",
+                        error=f"Error creating chart [{chart}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )

metadata/ingestion/source/dashboard/mstr/models.py

@@ -0,0 +1,144 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+MSTR Models
+"""
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class MstrOwner(BaseModel):
+    """
+    MSTR owner model
+    """
+
+    name: str
+    id: str
+
+
+class MstrProject(BaseModel):
+    """
+    MSTR project model
+    """
+
+    acg: int
+    id: str
+    name: str
+    status: int
+    alias: str
+    description: str
+    dateCreated: datetime
+    dateModified: datetime
+    owner: MstrOwner
+
+
+class MstrProjectList(BaseModel):
+    projects: Optional[List[MstrProject]]
+
+
+class MstrSearchResult(BaseModel):
+    """
+    Mstr Search Results model
+    """
+
+    name: str
+    id: str
+    type: int
+    description: Optional[str]
+    subtype: int
+    dateCreated: str
+    dateModified: str
+    version: str
+    acg: int
+    owner: MstrOwner
+    extType: int
+    viewMedia: int
+    certifiedInfo: dict
+    templateInfo: dict
+    projectId: str
+
+
+class MstrSearchResultList(BaseModel):
+    results: Optional[List[MstrSearchResult]]
+
+
+class MstrDashboard(BaseModel):
+    """
+    Mstr Dashboard model
+    """
+
+    name: str
+    id: str
+    type: int
+    description: Optional[str]
+    subtype: int
+    dateCreated: str
+    dateModified: str
+    version: str
+    acg: int
+    owner: MstrOwner
+    extType: int
+    viewMedia: int
+    certifiedInfo: dict
+    templateInfo: dict
+    projectId: str
+    projectName: str
+
+
+class MstrDashboardList(BaseModel):
+    dashboards: Optional[List[MstrDashboard]]
+
+
+class MstrAttribute(BaseModel):
+    id: str
+    type: int
+    name: str
+
+
+class MstrMetric(BaseModel):
+    id: str
+    type: int
+    name: str
+
+
+class MstrVisualization(BaseModel):
+    key: str
+    name: str
+    visualizationType: str
+
+
+class MstrPage(BaseModel):
+    key: str
+    name: str
+    visualizations: List[MstrVisualization]
+
+
+class MstrChapter(BaseModel):
+    key: str
+    name: str
+    pages: List[MstrPage]
+
+
+class MstrAvailableObject(BaseModel):
+    id: str
+    name: str
+    type: str
+
+
+class MstrDashboardDetails(BaseModel):
+    id: str
+    name: str
+    projectId: str
+    projectName: str
+    currentChapter: str
+    chapters: List[MstrChapter]

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -35,10 +35,13 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -324,14 +327,14 @@ class PowerbiSource(DashboardServiceSource):
                 project=self._fetch_dataset_workspace(dataset_id=dataset.id),
             )
             yield Either(right=data_model_request)
-            self.register_record_datamodel(datamodel_requst=data_model_request)
+            self.register_record_datamodel(datamodel_request=data_model_request)
 
         except Exception as exc:
             yield Either(
                 left=StackTraceError(
                     name=dataset.name,
                     error=f"Error yielding Data Model [{dataset.name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -405,9 +408,10 @@ class PowerbiSource(DashboardServiceSource):
                             service_name=self.context.dashboard_service,
                             chart_name=chart,
                         )
-                        for chart in self.context.charts
+                        for chart in self.context.charts or []
                     ],
                     service=self.context.dashboard_service,
+                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             else:
                 dashboard_request = CreateDashboardRequest(
@@ -420,6 +424,7 @@ class PowerbiSource(DashboardServiceSource):
                     project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.name,
                     service=self.context.dashboard_service,
+                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -428,7 +433,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Error creating dashboard [{dashboard_details}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -471,7 +476,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Lineage",
                     error=f"Error to yield report and dashboard lineage details: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -525,7 +530,7 @@ class PowerbiSource(DashboardServiceSource):
                         "Error to yield datamodel and report lineage details for DB "
                         f"service name [{db_service_name}]: {exc}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -563,7 +568,7 @@ class PowerbiSource(DashboardServiceSource):
                             "Error to yield datamodel lineage details for DB "
                             f"service name [{db_service_name}]: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -592,7 +597,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Dashboard Lineage",
                     error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -636,7 +641,7 @@ class PowerbiSource(DashboardServiceSource):
                         left=StackTraceError(
                             name=chart.title,
                             error=f"Error creating chart [{chart.title}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )