openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -32,11 +32,14 @@ from metadata.generated.schema.entity.services.connections.dashboard.domoDashboa
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     OpenMetadataConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -90,19 +93,14 @@ class DomodashboardSource(DashboardServiceSource):
     def get_dashboard_details(self, dashboard: DomoDashboardDetails) -> dict:
         return dashboard
 
-    def get_owner_details(
+    def get_owner_ref(
         self, dashboard_details: DomoDashboardDetails
     ) -> Optional[EntityReference]:
-        for owner in dashboard_details.owners:
+        for owner in dashboard_details.owners or []:
             try:
                 owner_details = self.client.domo.users_get(owner.id)
                 if owner_details.get("email"):
-                    user = self.metadata.get_user_by_email(owner_details["email"])
-                    if user:
-                        return EntityReference(id=user.id.__root__, type="user")
-                    logger.warning(
-                        f"No user found with email [{owner_details['email']}] in OMD"
-                    )
+                    return self.metadata.get_reference_by_email(owner_details["email"])
             except Exception as exc:
                 logger.warning(
                     f"Error while getting details of user {owner.displayName} - {exc}"
@@ -129,9 +127,10 @@ class DomodashboardSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -140,7 +139,7 @@ class DomodashboardSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Error extracting data from {dashboard_details.name} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except ValidationError as err:
@@ -148,7 +147,7 @@ class DomodashboardSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Error building pydantic model for {dashboard_details.name} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -156,7 +155,7 @@ class DomodashboardSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Wild error ingesting dashboard {dashboard_details.name} - {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -230,7 +229,7 @@ class DomodashboardSource(DashboardServiceSource):
                     left=StackTraceError(
                         name=name,
                         error=f"Error creating chart [{name}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -114,9 +114,10 @@ class LightdashSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield dashboard_request
             self.register_record(dashboard_request=dashboard_request)

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -8,6 +8,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+# pylint: disable=too-many-lines
 """
 Looker source module.
 Supports:
@@ -46,9 +47,7 @@ from metadata.generated.schema.api.data.createDashboardDataModel import (
 )
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
-from metadata.generated.schema.entity.data.dashboard import (
-    Dashboard as MetadataDashboard,
-)
+from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.data.dashboardDataModel import (
     DashboardDataModel,
     DataModelType,
@@ -62,6 +61,9 @@ from metadata.generated.schema.entity.services.dashboardService import (
     DashboardServiceType,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
@@ -75,7 +77,7 @@ from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDe
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.generated.schema.type.usageRequest import UsageRequest
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.parser import LineageParser
@@ -403,7 +405,7 @@ class LookerSource(DashboardServiceSource):
                     project=model.project_name,
                 )
                 yield Either(right=explore_datamodel)
-                self.register_record_datamodel(datamodel_requst=explore_datamodel)
+                self.register_record_datamodel(datamodel_request=explore_datamodel)
 
                 # build datamodel by our hand since ack_sink=False
                 self.context.dataModel = self._build_data_model(datamodel_name)
@@ -440,7 +442,7 @@ class LookerSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=model.name,
                     error=f"Validation error yielding Data Model [{model.name}]: {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as err:
@@ -448,7 +450,7 @@ class LookerSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=model.name,
                     error=f"Wild error yielding Data Model [{model.name}]: {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -504,14 +506,14 @@ class LookerSource(DashboardServiceSource):
                 self._view_data_model = self._build_data_model(
                     build_datamodel_name(explore.model_name, view.name)
                 )
-                self.register_record_datamodel(datamodel_requst=data_model_request)
+                self.register_record_datamodel(datamodel_request=data_model_request)
                 yield from self.add_view_lineage(view, explore)
             else:
                 yield Either(
                     left=StackTraceError(
                         name=view_name,
                         error=f"Cannot find the view [{view_name}]: empty",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -579,7 +581,7 @@ class LookerSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=view.name,
                     error=f"Error to yield lineage details for view [{view.name}]: {err}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -612,7 +614,7 @@ class LookerSource(DashboardServiceSource):
             dashboard_id=dashboard.id, fields=",".join(GET_DASHBOARD_FIELDS)
         )
 
-    def get_owner_details(
+    def get_owner_ref(
         self, dashboard_details: LookerDashboard
     ) -> Optional[EntityReference]:
         """Get dashboard owner
@@ -630,9 +632,7 @@ class LookerSource(DashboardServiceSource):
         try:
             if dashboard_details.user_id is not None:
                 dashboard_owner = self.client.user(dashboard_details.user_id)
-                user = self.metadata.get_user_by_email(dashboard_owner.email)
-                if user:
-                    return EntityReference(id=user.id.__root__, type="user")
+                return self.metadata.get_reference_by_email(dashboard_owner.email)
 
         except Exception as err:
             logger.debug(traceback.format_exc())
@@ -657,13 +657,14 @@ class LookerSource(DashboardServiceSource):
                     service_name=self.context.dashboard_service,
                     chart_name=chart,
                 )
-                for chart in self.context.charts
+                for chart in self.context.charts or []
             ],
             # Dashboards are created from the UI directly. They are not linked to a project
             # like LookML assets, but rather just organised in folders.
             project=self._get_dashboard_project(dashboard_details),
             sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/dashboards/{dashboard_details.id}",
             service=self.context.dashboard_service,
+            owner=self.get_owner_ref(dashboard_details=dashboard_details),
         )
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
@@ -756,12 +757,12 @@ class LookerSource(DashboardServiceSource):
                 if cached_explore:
                     dashboard_fqn = fqn.build(
                         self.metadata,
-                        entity_type=MetadataDashboard,
+                        entity_type=Dashboard,
                         service_name=self.context.dashboard_service,
                         dashboard_name=self.context.dashboard,
                     )
                     dashboard_entity = self.metadata.get_by_name(
-                        entity=MetadataDashboard, fqn=dashboard_fqn
+                        entity=Dashboard, fqn=dashboard_fqn
                     )
                     yield Either(
                         right=AddLineageRequest(
@@ -786,7 +787,7 @@ class LookerSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_entity.displayName,
                     error=f"Unexpected exception yielding lineage from [{dashboard_entity.displayName}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -794,7 +795,7 @@ class LookerSource(DashboardServiceSource):
         self,
         source: str,
         db_service_name: str,
-        to_entity: Union[MetadataDashboard, DashboardDataModel],
+        to_entity: Union[Dashboard, DashboardDataModel],
     ) -> Optional[Either[AddLineageRequest]]:
         """
         Once we have a list of origin data sources, check their components
@@ -832,7 +833,6 @@ class LookerSource(DashboardServiceSource):
                     to_entity.id.__root__
                     not in self._added_lineage[from_entity.id.__root__]
                 ):
-
                     self._added_lineage[from_entity.id.__root__].append(
                         to_entity.id.__root__
                     )
@@ -879,7 +879,7 @@ class LookerSource(DashboardServiceSource):
                     left=StackTraceError(
                         name=chart.id,
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -939,9 +939,22 @@ class LookerSource(DashboardServiceSource):
         :return: UsageRequest, if not computed
         """
 
-        dashboard: MetadataDashboard = self.context.dashboard
+        dashboard_name = self.context.dashboard
 
         try:
+            dashboard_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=Dashboard,
+                service_name=self.context.dashboard_service,
+                dashboard_name=dashboard_name,
+            )
+
+            dashboard: Dashboard = self.metadata.get_by_name(
+                entity=Dashboard,
+                fqn=dashboard_fqn,
+                fields=["usageSummary"],
+            )
+
             current_views = dashboard_details.view_count
 
             if not current_views:
@@ -993,8 +1006,8 @@ class LookerSource(DashboardServiceSource):
         except Exception as exc:
             yield Either(
                 left=StackTraceError(
-                    name=f"{dashboard.name} Usage",
-                    error=f"Exception computing dashboard usage for {dashboard.fullyQualifiedName.__root__}: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    name=f"{dashboard_name} Usage",
+                    error=f"Exception computing dashboard usage for {dashboard_name}: {exc}",
+                    stackTrace=traceback.format_exc(),
                 )
             )

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -26,11 +26,16 @@ from metadata.generated.schema.entity.services.connections.dashboard.metabaseCon
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     OpenMetadataConnection,
 )
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.lineage.sql_lineage import search_table_entities
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -147,9 +152,10 @@ class MetabaseSource(DashboardServiceSource):
                         service_name=self.context.dashboard_service,
                         chart_name=chart,
                     )
-                    for chart in self.context.charts
+                    for chart in self.context.charts or []
                 ],
                 service=self.context.dashboard_service,
+                owner=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -158,7 +164,7 @@ class MetabaseSource(DashboardServiceSource):
                 left=StackTraceError(
                     name=dashboard_details.name,
                     error=f"Error creating dashboard [{dashboard_details.name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -202,7 +208,7 @@ class MetabaseSource(DashboardServiceSource):
                     left=StackTraceError(
                         name="Chart",
                         error=f"Error creating chart [{chart}]: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
@@ -253,10 +259,13 @@ class MetabaseSource(DashboardServiceSource):
                     left=StackTraceError(
                         name="Lineage",
                         error=f"Error adding lineage: {exc}",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
+    def _get_database_service(self, db_service_name: str):
+        return self.metadata.get_by_name(DatabaseService, db_service_name)
+
     def _yield_lineage_from_query(
         self, chart_details: MetabaseChart, db_service_name: str, dashboard_name: str
     ) -> Iterable[Either[AddLineageRequest]]:
@@ -275,7 +284,15 @@ class MetabaseSource(DashboardServiceSource):
 
         database_name = database.details.db if database and database.details else None
 
-        lineage_parser = LineageParser(query)
+        db_service = self._get_database_service(db_service_name)
+
+        lineage_parser = LineageParser(
+            query,
+            ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
+            if db_service
+            else None,
+        )
+
         for table in lineage_parser.source_tables:
             database_schema_name, table = fqn.split(str(table))[-2:]
             database_schema_name = self.check_database_schema_name(database_schema_name)

metadata/ingestion/source/dashboard/mode/client.py

@@ -74,23 +74,18 @@ class ModeApiClient:
         Returns:
             dict
         """
-        try:
-            all_reports = []
-            response_collections = self.client.get(f"/{workspace_name}/{COLLECTIONS}")
-            collections = response_collections[EMBEDDED]["spaces"]
-            for collection in collections:
-                response_reports = self.get_all_reports_for_collection(
-                    workspace_name=workspace_name,
-                    collection_token=collection.get(TOKEN),
-                )
+        all_reports = []
+        response_collections = self.client.get(f"/{workspace_name}/{COLLECTIONS}")
+        collections = response_collections[EMBEDDED]["spaces"]
+        for collection in collections:
+            response_reports = self.get_all_reports_for_collection(
+                workspace_name=workspace_name,
+                collection_token=collection.get(TOKEN),
+            )
+            if response_reports:
                 reports = response_reports[EMBEDDED][REPORTS]
                 all_reports.extend(reports)
-            return all_reports
-        except Exception as exc:  # pylint: disable=broad-except
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Error fetching all reports: {exc}")
-
-        return None
+        return all_reports
 
     def get_all_reports_for_collection(
         self, workspace_name: str, collection_token: str
@@ -180,11 +175,3 @@ class ModeApiClient:
             logger.warning(f"Error fetching all data sources: {exc}")
 
         return None
-
-    def get_user_account(self) -> dict:
-        """Method to fetch account details
-        Returns:
-            dict
-        """
-        response = self.client.get("/account")
-        return response

metadata/ingestion/source/dashboard/mode/connection.py

@@ -12,6 +12,7 @@
 """
 Source connection handler
 """
+from functools import partial
 from typing import Optional
 
 from metadata.generated.schema.entity.automations.workflow import (
@@ -43,7 +44,11 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_fn = {"CheckAccess": client.get_user_account}
+    test_fn = {
+        "CheckDashboards": partial(
+            client.fetch_all_reports, service_connection.workspaceName
+        )
+    }
 
     test_connection_steps(
         metadata=metadata,

metadata/ingestion/source/dashboard/mode/metadata.py

@@ -23,10 +23,13 @@ from metadata.generated.schema.entity.data.dashboard import (
 from metadata.generated.schema.entity.services.connections.dashboard.modeConnection import (
     ModeConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.lineage.sql_lineage import search_table_entities
@@ -103,9 +106,10 @@ class ModeSource(DashboardServiceSource):
                     service_name=self.context.dashboard_service,
                     chart_name=chart,
                 )
-                for chart in self.context.charts
+                for chart in self.context.charts or []
             ],
             service=self.context.dashboard_service,
+            owner=self.get_owner_ref(dashboard_details=dashboard_details),
         )
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
@@ -157,7 +161,7 @@ class ModeSource(DashboardServiceSource):
                 left=StackTraceError(
                     name="Lineage",
                     error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
 
@@ -208,6 +212,6 @@ class ModeSource(DashboardServiceSource):
                         left=StackTraceError(
                             name=name,
                             error=f"Error to yield dashboard chart [{chart}]: {exc}",
-                            stack_trace=traceback.format_exc(),
+                            stackTrace=traceback.format_exc(),
                         )
                     )