openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/qliksense/metadata.py

@@ -35,10 +35,13 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -126,9 +129,10 @@ class QliksenseSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -137,7 +141,7 @@ class QliksenseSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.qDocName,
                     error=f"Error creating dashboard [{dashboard_details.qDocName}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -177,7 +181,7 @@ class QliksenseSource(DashboardServiceSource):
                     left=StackTraceError(
                         name=dashboard_details.qDocName,
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -221,7 +225,7 @@ class QliksenseSource(DashboardServiceSource):
                         columns=self.get_column_info(data_model),
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
                 except Exception as exc:
                     name = (
                         data_model.tableName if data_model.tableName else data_model.id
@@ -230,7 +234,7 @@ class QliksenseSource(DashboardServiceSource):
                         left=StackTraceError(
                             name=name,
                             error=f"Error yielding Data Model [{name}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )
 
@@ -314,7 +318,7 @@ class QliksenseSource(DashboardServiceSource):
                             "Error to yield dashboard lineage details for DB "
                             f"service name [{db_service_name}]: {err}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/dashboard/quicksight/metadata.py

@@ -24,10 +24,13 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.quickSightConnection import (
     QuickSightConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -141,9 +144,10 @@ class QuicksightSource(DashboardServiceSource):
                     service_name=self.context.dashboard_service,
                     chart_name=chart,
                 )
-                for chart in self.context.charts
+                for chart in self.context.charts or []
             ],
             service=self.context.dashboard_service,
+            owner=self.get_owner_ref(dashboard_details=dashboard_details),
         )
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
@@ -182,7 +186,7 @@ class QuicksightSource(DashboardServiceSource):
                     left=StackTraceError(
                         name="Chart",
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -233,7 +237,7 @@ class QuicksightSource(DashboardServiceSource):
                                     "Error to yield dashboard lineage details for DB service"
                                     f" name [{db_service_name}]: {err}"
                                 ),
-                                stack_trace=traceback.format_exc(),
+                                stackTrace=traceback.format_exc(),
                             )
                         )
 
@@ -292,6 +296,6 @@ class QuicksightSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Lineage",
                     error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )

metadata/ingestion/source/dashboard/redash/metadata.py

@@ -27,11 +27,14 @@ from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.redashConnection import (
     RedashConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -84,7 +87,7 @@ class RedashSource(DashboardServiceSource):
             for dashboard in self.dashboard_list:
                 self.tags.extend(dashboard.get("tags") or [])
 
-    def yield_tag(self, *_, **__) -> Iterable[Either[OMetaTagAndClassification]]:
+    def yield_bulk_tags(self, *_, **__) -> Iterable[Either[OMetaTagAndClassification]]:
         """Fetch Dashboard Tags"""
         yield from get_ometa_tag_and_classification(
             tags=self.tags,
@@ -103,14 +106,19 @@ class RedashSource(DashboardServiceSource):
     def get_dashboard_details(self, dashboard: dict) -> dict:
         return self.client.get_dashboard(dashboard["slug"])
 
-    def get_owner_details(self, dashboard_details) -> Optional[EntityReference]:
-        """Get owner from mail"""
-        if dashboard_details.get("user") and dashboard_details["user"].get("email"):
-            user = self.metadata.get_user_by_email(
-                dashboard_details["user"].get("email")
-            )
-            if user:
-                return EntityReference(id=user.id.__root__, type="user")
+    def get_owner_ref(self, dashboard_details) -> Optional[EntityReference]:
+        """
+        Get owner from email
+        """
+        try:
+            if dashboard_details.get("user") and dashboard_details["user"].get("email"):
+                return self.metadata.get_reference_by_email(
+                    dashboard_details["user"].get("email")
+                )
+            return None
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
     def get_dashboard_url(self, dashboard_details: dict) -> str:
@@ -149,7 +157,7 @@ class RedashSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
                 sourceUrl=self.get_dashboard_url(dashboard_details),
@@ -159,6 +167,7 @@ class RedashSource(DashboardServiceSource):
                     classification_name=REDASH_TAG_CATEGORY,
                     include_tags=self.source_config.includeTags,
                 ),
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -168,7 +177,7 @@ class RedashSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Dashboard",
                     error=f"Error to yield dashboard for {dashboard_details}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -229,7 +238,7 @@ class RedashSource(DashboardServiceSource):
                             "Error to yield dashboard lineage details for DB "
                             f"service name [{db_service_name}]: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -272,6 +281,6 @@ class RedashSource(DashboardServiceSource):
                             "Error to yield dashboard chart for widget_id: "
                             f"{widgets['id']} and {dashboard_details}: {exc}"
                         ),
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )

metadata/ingestion/source/dashboard/superset/api_source.py

@@ -24,7 +24,10 @@ from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin
 from metadata.ingestion.source.dashboard.superset.models import (
     ChartResult,
@@ -94,9 +97,10 @@ class SupersetAPISource(SupersetSourceMixin):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -105,7 +109,7 @@ class SupersetAPISource(SupersetSourceMixin):
                 left=StackTraceError(
                     name=dashboard_details.id or "Dashboard",
                     error=f"Error creating dashboard [{dashboard_details.dashboard_title}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -144,7 +148,7 @@ class SupersetAPISource(SupersetSourceMixin):
                     left=StackTraceError(
                         name=chart_json.id,
                         error=f"Error creating chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -188,12 +192,11 @@ class SupersetAPISource(SupersetSourceMixin):
     def yield_datamodel(
         self, dashboard_details: DashboardResult
     ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
-
         if self.source_config.includeDataModels:
             for chart_id in self._get_charts_of_dashboard(dashboard_details):
                 try:
                     chart_json = self.all_charts.get(chart_id)
-                    if not chart_json:
+                    if not chart_json or not chart_json.datasource_id:
                         logger.warning(
                             f"chart details for id: {chart_id} not found, skipped"
                         )
@@ -217,12 +220,12 @@ class SupersetAPISource(SupersetSourceMixin):
                         dataModelType=DataModelType.SupersetDataModel.value,
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
                 except Exception as exc:
                     yield Either(
                         left=StackTraceError(
                             name=f"{dashboard_details.id} DataModel",
                             error=f"Error yielding Data Model [{dashboard_details.id}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )

metadata/ingestion/source/dashboard/superset/client.py

@@ -99,6 +99,13 @@ class SupersetAPIClient:
         )
         self.client = REST(client_config)
 
+    def get_dashboard_count(self) -> int:
+        resp_dashboards = self.client.get("/dashboard/?q=(page:0,page_size:1)")
+        if resp_dashboards:
+            dashboard_count = SupersetDashboardCount(**resp_dashboards)
+            return dashboard_count.count
+        return 0
+
     def fetch_total_dashboards(self) -> int:
         """
         Fetch total dashboard
@@ -107,10 +114,7 @@ class SupersetAPIClient:
             int
         """
         try:
-            resp_dashboards = self.client.get("/dashboard/?q=(page:0,page_size:1)")
-            if resp_dashboards:
-                dashboard_count = SupersetDashboardCount(**resp_dashboards)
-                return dashboard_count.count
+            return self.get_dashboard_count()
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the dashboard count")
@@ -142,6 +146,13 @@ class SupersetAPIClient:
             logger.warning("Failed to fetch the dashboard list")
         return SupersetDashboardCount()
 
+    def get_chart_count(self) -> int:
+        resp_chart = self.client.get("/chart/?q=(page:0,page_size:1)")
+        if resp_chart:
+            chart_count = SupersetChart(**resp_chart)
+            return chart_count.count
+        return 0
+
     def fetch_total_charts(self) -> int:
         """
         Fetch the total number of charts
@@ -149,12 +160,8 @@ class SupersetAPIClient:
         Returns:
              int
         """
-
         try:
-            resp_chart = self.client.get("/chart/?q=(page:0,page_size:1)")
-            if resp_chart:
-                chart_count = SupersetChart(**resp_chart)
-                return chart_count.count
+            return self.get_chart_count()
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the chart count")

metadata/ingestion/source/dashboard/superset/connection.py

@@ -78,9 +78,9 @@ def test_connection(
     test_fn = {}
 
     if isinstance(client, SupersetAPIClient):
-        test_fn["CheckAccess"] = client.fetch_total_dashboards
-        test_fn["GetDashboards"] = client.fetch_total_dashboards
-        test_fn["GetCharts"] = client.fetch_total_charts
+        test_fn["CheckAccess"] = client.get_dashboard_count
+        test_fn["GetDashboards"] = client.get_dashboard_count
+        test_fn["GetCharts"] = client.get_chart_count
     else:
         test_fn["CheckAccess"] = partial(test_connection_engine_step, client)
         test_fn["GetDashboards"] = partial(test_query, client, FETCH_DASHBOARDS_TEST)

metadata/ingestion/source/dashboard/superset/db_source.py

@@ -28,10 +28,13 @@ from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin
 from metadata.ingestion.source.dashboard.superset.models import (
@@ -83,8 +86,9 @@ class SupersetDBSource(SupersetSourceMixin):
     def get_column_list(self, table_name: str) -> Iterable[FetchChart]:
         try:
             if table_name:
-                sql_query = sql.text(FETCH_COLUMN.format(table_name=table_name.lower()))
-                col_list = self.engine.execute(sql_query)
+                col_list = self.engine.execute(
+                    sql.text(FETCH_COLUMN), table_name=table_name.lower()
+                )
                 return [FetchColumn(**col) for col in col_list]
         except Exception as err:
             logger.debug(traceback.format_exc())
@@ -117,9 +121,10 @@ class SupersetDBSource(SupersetSourceMixin):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -131,7 +136,7 @@ class SupersetDBSource(SupersetSourceMixin):
                         f"Error yielding Dashboard [{dashboard_details.id} "
                         f"- {dashboard_details.dashboard_title}]: {exc}"
                     ),
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -173,7 +178,7 @@ class SupersetDBSource(SupersetSourceMixin):
                     left=StackTraceError(
                         name=chart_json.id,
                         error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -211,11 +216,10 @@ class SupersetDBSource(SupersetSourceMixin):
     def yield_datamodel(
         self, dashboard_details: FetchDashboard
     ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
-
         if self.source_config.includeDataModels:
             for chart_id in self._get_charts_of_dashboard(dashboard_details):
                 chart_json = self.all_charts.get(chart_id)
-                if not chart_json:
+                if not chart_json or not chart_json.datasource_id:
                     logger.warning(
                         f"chart details for id: {chart_id} not found, skipped"
                     )
@@ -236,13 +240,13 @@ class SupersetDBSource(SupersetSourceMixin):
                         dataModelType=DataModelType.SupersetDataModel.value,
                     )
                     yield Either(right=data_model_request)
-                    self.register_record_datamodel(datamodel_requst=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
 
                 except Exception as exc:
                     yield Either(
                         left=StackTraceError(
                             name=chart_json.table_name,
                             error=f"Error yielding Data Model [{chart_json.table_name}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -28,11 +28,14 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -94,25 +97,26 @@ class SupersetSourceMixin(DashboardServiceSource):
 
     def _get_user_by_email(self, email: Optional[str]) -> Optional[EntityReference]:
         if email:
-            user = self.metadata.get_user_by_email(email)
-            if user:
-                return EntityReference(id=user.id.__root__, type="user")
-
+            return self.metadata.get_reference_by_email(email)
         return None
 
-    def get_owner_details(
+    def get_owner_ref(
         self, dashboard_details: Union[DashboardResult, FetchDashboard]
     ) -> EntityReference:
-        if hasattr(dashboard_details, "owner"):
-            for owner in dashboard_details.owners or []:
-                if owner.email:
-                    user = self._get_user_by_email(owner.email)
-                    if user:
-                        return user
-        if dashboard_details.email:
-            user = self._get_user_by_email(dashboard_details.email)
-            if user:
-                return user
+        try:
+            if hasattr(dashboard_details, "owners"):
+                for owner in dashboard_details.owners or []:
+                    if owner.email:
+                        user = self._get_user_by_email(owner.email)
+                        if user:
+                            return user
+            if dashboard_details.email:
+                user = self._get_user_by_email(dashboard_details.email)
+                if user:
+                    return user
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
     def _get_charts_of_dashboard(
@@ -185,7 +189,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                                     "Error to yield dashboard lineage details for DB "
                                     f"service name [{db_service_name}]: {exc}"
                                 ),
-                                stack_trace=traceback.format_exc(),
+                                stackTrace=traceback.format_exc(),
                             )
                         )
 
@@ -230,7 +234,7 @@ class SupersetSourceMixin(DashboardServiceSource):
                         name=field.id,
                         displayName=field.column_name,
                         description=field.description,
-                        dataLength=col_parse.get("dataLength", 0),
+                        dataLength=int(col_parse.get("dataLength", 0)),
                     )
                     datasource_columns.append(parsed_fields)
             except Exception as exc:

metadata/ingestion/source/dashboard/superset/queries.py

@@ -85,5 +85,5 @@ inner join
 on 
 	t.id=tc.table_id  
 where 
-	table_name= '{table_name}'
+	table_name= %(table_name)s
 """