openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/ingestion/source/dashboard/mstr/models.py

@@ -0,0 +1,144 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+MSTR Models
+"""
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+
+class MstrOwner(BaseModel):
+    """
+    MSTR owner model
+    """
+
+    name: str
+    id: str
+
+
+class MstrProject(BaseModel):
+    """
+    MSTR project model
+    """
+
+    acg: int
+    id: str
+    name: str
+    status: int
+    alias: str
+    description: str
+    dateCreated: datetime
+    dateModified: datetime
+    owner: MstrOwner
+
+
+class MstrProjectList(BaseModel):
+    projects: Optional[List[MstrProject]]
+
+
+class MstrSearchResult(BaseModel):
+    """
+    Mstr Search Results model
+    """
+
+    name: str
+    id: str
+    type: int
+    description: Optional[str]
+    subtype: int
+    dateCreated: str
+    dateModified: str
+    version: str
+    acg: int
+    owner: MstrOwner
+    extType: int
+    viewMedia: int
+    certifiedInfo: dict
+    templateInfo: dict
+    projectId: str
+
+
+class MstrSearchResultList(BaseModel):
+    results: Optional[List[MstrSearchResult]]
+
+
+class MstrDashboard(BaseModel):
+    """
+    Mstr Dashboard model
+    """
+
+    name: str
+    id: str
+    type: int
+    description: Optional[str]
+    subtype: int
+    dateCreated: str
+    dateModified: str
+    version: str
+    acg: int
+    owner: MstrOwner
+    extType: int
+    viewMedia: int
+    certifiedInfo: dict
+    templateInfo: dict
+    projectId: str
+    projectName: str
+
+
+class MstrDashboardList(BaseModel):
+    dashboards: Optional[List[MstrDashboard]]
+
+
+class MstrAttribute(BaseModel):
+    id: str
+    type: int
+    name: str
+
+
+class MstrMetric(BaseModel):
+    id: str
+    type: int
+    name: str
+
+
+class MstrVisualization(BaseModel):
+    key: str
+    name: str
+    visualizationType: str
+
+
+class MstrPage(BaseModel):
+    key: str
+    name: str
+    visualizations: List[MstrVisualization]
+
+
+class MstrChapter(BaseModel):
+    key: str
+    name: str
+    pages: List[MstrPage]
+
+
+class MstrAvailableObject(BaseModel):
+    id: str
+    name: str
+    type: str
+
+
+class MstrDashboardDetails(BaseModel):
+    id: str
+    name: str
+    projectId: str
+    projectName: str
+    currentChapter: str
+    chapters: List[MstrChapter]

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -19,6 +19,9 @@ from typing import List, Optional, Tuple
 
 import msal
 
+from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
+    PowerBIConnection,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.powerbi.models import (
@@ -52,7 +55,7 @@ class PowerBiApiClient:
 
     client: REST
 
-    def __init__(self, config):
+    def __init__(self, config: PowerBIConnection):
         self.config = config
         self.msal_client = msal.ConfidentialClientApplication(
             client_id=self.config.clientId,

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -35,10 +35,13 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -324,14 +327,14 @@ class PowerbiSource(DashboardServiceSource):
                 project=self._fetch_dataset_workspace(dataset_id=dataset.id),
             )
             yield Either(right=data_model_request)
-            self.register_record_datamodel(datamodel_requst=data_model_request)
+            self.register_record_datamodel(datamodel_request=data_model_request)
 
         except Exception as exc:
             yield Either(
                 left=StackTraceError(
                     name=dataset.name,
                     error=f"Error yielding Data Model [{dataset.name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -405,9 +408,10 @@ class PowerbiSource(DashboardServiceSource):
                             service_name=self.context.dashboard_service,
                             chart_name=chart,
                         )
-                        for chart in self.context.charts
+                        for chart in self.context.charts or []
                     ],
                     service=self.context.dashboard_service,
+                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             else:
                 dashboard_request = CreateDashboardRequest(
@@ -420,6 +424,7 @@ class PowerbiSource(DashboardServiceSource):
                     project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.name,
                     service=self.context.dashboard_service,
+                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -428,7 +433,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Error creating dashboard [{dashboard_details}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -471,7 +476,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Lineage",
                     error=f"Error to yield report and dashboard lineage details: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -525,7 +530,7 @@ class PowerbiSource(DashboardServiceSource):
                         "Error to yield datamodel and report lineage details for DB "
                         f"service name [{db_service_name}]: {exc}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -563,7 +568,7 @@ class PowerbiSource(DashboardServiceSource):
                             "Error to yield datamodel lineage details for DB "
                             f"service name [{db_service_name}]: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -592,7 +597,7 @@ class PowerbiSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Dashboard Lineage",
                     error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -636,7 +641,7 @@ class PowerbiSource(DashboardServiceSource):
                         left=StackTraceError(
                             name=chart.title,
                             error=f"Error creating chart [{chart.title}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )
 

metadata/ingestion/source/dashboard/qliksense/metadata.py

@@ -35,10 +35,13 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -126,9 +129,10 @@ class QliksenseSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -137,7 +141,7 @@ class QliksenseSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.qDocName,
                     error=f"Error creating dashboard [{dashboard_details.qDocName}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -177,7 +181,7 @@ class QliksenseSource(DashboardServiceSource):
                     left=StackTraceError(
                         name=dashboard_details.qDocName,
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -221,7 +225,7 @@ class QliksenseSource(DashboardServiceSource):
                         columns=self.get_column_info(data_model),
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
                 except Exception as exc:
                     name = (
                         data_model.tableName if data_model.tableName else data_model.id
@@ -230,7 +234,7 @@ class QliksenseSource(DashboardServiceSource):
                         left=StackTraceError(
                             name=name,
                             error=f"Error yielding Data Model [{name}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )
 
@@ -314,7 +318,7 @@ class QliksenseSource(DashboardServiceSource):
                             "Error to yield dashboard lineage details for DB "
                             f"service name [{db_service_name}]: {err}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/dashboard/quicksight/metadata.py

@@ -24,10 +24,13 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.quickSightConnection import (
     QuickSightConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -141,9 +144,10 @@ class QuicksightSource(DashboardServiceSource):
                     service_name=self.context.dashboard_service,
                     chart_name=chart,
                 )
-                for chart in self.context.charts
+                for chart in self.context.charts or []
             ],
             service=self.context.dashboard_service,
+            owner=self.get_owner_ref(dashboard_details=dashboard_details),
         )
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
@@ -182,7 +186,7 @@ class QuicksightSource(DashboardServiceSource):
                     left=StackTraceError(
                         name="Chart",
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -233,7 +237,7 @@ class QuicksightSource(DashboardServiceSource):
                                     "Error to yield dashboard lineage details for DB service"
                                     f" name [{db_service_name}]: {err}"
                                 ),
-                                stack_trace=traceback.format_exc(),
+                                stackTrace=traceback.format_exc(),
                             )
                         )
 
@@ -292,6 +296,6 @@ class QuicksightSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Lineage",
                     error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -27,11 +27,14 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.redashConnection import (
     RedashConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -84,7 +87,7 @@ class RedashSource(DashboardServiceSource):
             for dashboard in self.dashboard_list:
                 self.tags.extend(dashboard.get("tags") or [])
 
-    def yield_tag(self, *_, **__) -> Iterable[Either[OMetaTagAndClassification]]:
+    def yield_bulk_tags(self, *_, **__) -> Iterable[Either[OMetaTagAndClassification]]:
         """Fetch Dashboard Tags"""
         yield from get_ometa_tag_and_classification(
             tags=self.tags,
@@ -103,14 +106,19 @@ class RedashSource(DashboardServiceSource):
     def get_dashboard_details(self, dashboard: dict) -> dict:
         return self.client.get_dashboard(dashboard["slug"])
 
-    def get_owner_details(self, dashboard_details) -> Optional[EntityReference]:
-        """Get owner from mail"""
-        if dashboard_details.get("user") and dashboard_details["user"].get("email"):
-            user = self.metadata.get_user_by_email(
-                dashboard_details["user"].get("email")
-            )
-            if user:
-                return EntityReference(id=user.id.__root__, type="user")
+    def get_owner_ref(self, dashboard_details) -> Optional[EntityReference]:
+        """
+        Get owner from email
+        """
+        try:
+            if dashboard_details.get("user") and dashboard_details["user"].get("email"):
+                return self.metadata.get_reference_by_email(
+                    dashboard_details["user"].get("email")
+                )
+            return None
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
     def get_dashboard_url(self, dashboard_details: dict) -> str:
@@ -149,7 +157,7 @@ class RedashSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
                 sourceUrl=self.get_dashboard_url(dashboard_details),
@@ -159,6 +167,7 @@ class RedashSource(DashboardServiceSource):
                     classification_name=REDASH_TAG_CATEGORY,
                     include_tags=self.source_config.includeTags,
                 ),
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -168,7 +177,7 @@ class RedashSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Dashboard",
                     error=f"Error to yield dashboard for {dashboard_details}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -229,7 +238,7 @@ class RedashSource(DashboardServiceSource):
                             "Error to yield dashboard lineage details for DB "
                             f"service name [{db_service_name}]: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -272,6 +281,6 @@ class RedashSource(DashboardServiceSource):
                             "Error to yield dashboard chart for widget_id: "
                             f"{widgets['id']} and {dashboard_details}: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )

metadata/ingestion/source/dashboard/superset/api_source.py

@@ -24,7 +24,10 @@ from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin
 from metadata.ingestion.source.dashboard.superset.models import (
     ChartResult,
@@ -94,9 +97,10 @@ class SupersetAPISource(SupersetSourceMixin):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -105,7 +109,7 @@ class SupersetAPISource(SupersetSourceMixin):
                 left=StackTraceError(
                     name=dashboard_details.id or "Dashboard",
                     error=f"Error creating dashboard [{dashboard_details.dashboard_title}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -144,7 +148,7 @@ class SupersetAPISource(SupersetSourceMixin):
                     left=StackTraceError(
                         name=chart_json.id,
                         error=f"Error creating chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -188,12 +192,11 @@ class SupersetAPISource(SupersetSourceMixin):
     def yield_datamodel(
         self, dashboard_details: DashboardResult
     ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
-
         if self.source_config.includeDataModels:
             for chart_id in self._get_charts_of_dashboard(dashboard_details):
                 try:
                     chart_json = self.all_charts.get(chart_id)
-                    if not chart_json:
+                    if not chart_json or not chart_json.datasource_id:
                         logger.warning(
                             f"chart details for id: {chart_id} not found, skipped"
                         )
@@ -217,12 +220,12 @@ class SupersetAPISource(SupersetSourceMixin):
                         dataModelType=DataModelType.SupersetDataModel.value,
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
                 except Exception as exc:
                     yield Either(
                         left=StackTraceError(
                             name=f"{dashboard_details.id} DataModel",
                             error=f"Error yielding Data Model [{dashboard_details.id}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )

metadata/ingestion/source/dashboard/superset/client.py

@@ -99,6 +99,13 @@ class SupersetAPIClient:
         )
         self.client = REST(client_config)
 
+    def get_dashboard_count(self) -> int:
+        resp_dashboards = self.client.get("/dashboard/?q=(page:0,page_size:1)")
+        if resp_dashboards:
+            dashboard_count = SupersetDashboardCount(**resp_dashboards)
+            return dashboard_count.count
+        return 0
+
     def fetch_total_dashboards(self) -> int:
         """
         Fetch total dashboard
@@ -107,10 +114,7 @@ class SupersetAPIClient:
             int
         """
         try:
-            resp_dashboards = self.client.get("/dashboard/?q=(page:0,page_size:1)")
-            if resp_dashboards:
-                dashboard_count = SupersetDashboardCount(**resp_dashboards)
-                return dashboard_count.count
+            return self.get_dashboard_count()
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the dashboard count")
@@ -142,6 +146,13 @@ class SupersetAPIClient:
             logger.warning("Failed to fetch the dashboard list")
         return SupersetDashboardCount()
 
+    def get_chart_count(self) -> int:
+        resp_chart = self.client.get("/chart/?q=(page:0,page_size:1)")
+        if resp_chart:
+            chart_count = SupersetChart(**resp_chart)
+            return chart_count.count
+        return 0
+
     def fetch_total_charts(self) -> int:
         """
         Fetch the total number of charts
@@ -149,12 +160,8 @@ class SupersetAPIClient:
         Returns:
              int
         """
-
         try:
-            resp_chart = self.client.get("/chart/?q=(page:0,page_size:1)")
-            if resp_chart:
-                chart_count = SupersetChart(**resp_chart)
-                return chart_count.count
+            return self.get_chart_count()
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the chart count")

metadata/ingestion/source/dashboard/superset/connection.py

@@ -78,9 +78,9 @@ def test_connection(
     test_fn = {}
 
     if isinstance(client, SupersetAPIClient):
-        test_fn["CheckAccess"] = client.fetch_total_dashboards
-        test_fn["GetDashboards"] = client.fetch_total_dashboards
-        test_fn["GetCharts"] = client.fetch_total_charts
+        test_fn["CheckAccess"] = client.get_dashboard_count
+        test_fn["GetDashboards"] = client.get_dashboard_count
+        test_fn["GetCharts"] = client.get_chart_count
     else:
         test_fn["CheckAccess"] = partial(test_connection_engine_step, client)
         test_fn["GetDashboards"] = partial(test_query, client, FETCH_DASHBOARDS_TEST)