openmetadata-ingestion 1.6.7.0__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/ingestion/sink/metadata_rest.py

@@ -34,7 +34,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
 from metadata.generated.schema.dataInsight.kpi.basic import KpiResult
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.generated.schema.entity.data.dashboard import Dashboard
-from metadata.generated.schema.entity.data.pipeline import PipelineStatus
+from metadata.generated.schema.entity.data.pipeline import Pipeline, PipelineStatus
 from metadata.generated.schema.entity.data.searchIndex import (
     SearchIndex,
     SearchIndexSampleData,
@@ -83,6 +83,7 @@ from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardUsage
 from metadata.ingestion.source.database.database_service import DataModelLink
+from metadata.ingestion.source.pipeline.pipeline_service import PipelineUsage
 from metadata.profiler.api.models import ProfilerResponse
 from metadata.sampler.models import SamplerResponse
 from metadata.utils.execution_time_tracker import calculate_execution_time
@@ -250,6 +251,13 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
     @_run_dispatch.register
     def write_lineage(self, add_lineage: AddLineageRequest) -> Either[Dict[str, Any]]:
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
+        if created_lineage.get("error"):
+            return Either(
+                left=StackTraceError(
+                    name="AddLineageRequestError", error=created_lineage["error"]
+                )
+            )
+
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
     @_run_dispatch.register
@@ -634,6 +642,18 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
 
         return Either(right=record)
 
+    @_run_dispatch.register
+    def write_pipeline_usage(self, pipeline_usage: PipelineUsage) -> Either[Pipeline]:
+        """
+        Send a UsageRequest update to a pipeline entity
+        :param pipeline_usage: pipeline entity and usage request
+        """
+        self.metadata.publish_pipeline_usage(
+            pipeline=pipeline_usage.pipeline,
+            pipeline_usage_request=pipeline_usage.usage,
+        )
+        return Either(right=pipeline_usage.pipeline)
+
     def close(self):
         """
         We don't have anything to close since we are using the given metadata client

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -198,6 +198,9 @@ class DashboardServiceTopology(ServiceTopology):
     )
 
 
+from metadata.utils.helpers import retry_with_docker_host
+
+
 # pylint: disable=too-many-public-methods
 class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     """
@@ -216,6 +219,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     dashboard_source_state: Set = set()
     datamodel_source_state: Set = set()
 
+    @retry_with_docker_host()
     def __init__(
         self,
         config: WorkflowSource,
@@ -248,7 +252,9 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     @abstractmethod
     def yield_dashboard_lineage_details(
-        self, dashboard_details: Any, db_service_name: str
+        self,
+        dashboard_details: Any,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -368,6 +374,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 yield lineage
 
         db_service_names = self.get_db_service_names()
+        if not db_service_names:
+            yield from self.yield_dashboard_lineage_details(dashboard_details) or []
         for db_service_name in db_service_names or []:
             yield from self.yield_dashboard_lineage_details(
                 dashboard_details, db_service_name

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -248,6 +248,8 @@ class DomodashboardSource(DashboardServiceSource):
                 )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict, db_service_name
+        self,
+        dashboard_details: dict,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """No lineage implemented"""

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -176,7 +176,7 @@ class LightdashSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LightdashDashboard,
-        db_service_name: Optional[str],
+        db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -185,7 +185,7 @@ class LookerSource(DashboardServiceSource):
     Its client uses Looker 40 from the SDK: client = looker_sdk.init40()
     """
 
-    # pylint: disable=too-many-instance-attributes
+    # pylint: disable=too-many-instance-attributes, too-many-public-methods
 
     config: WorkflowSource
     metadata: OpenMetadata
@@ -924,7 +924,9 @@ class LookerSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: LookerDashboard, _: str
+        self,
+        dashboard_details: LookerDashboard,
+        _: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -1196,3 +1198,7 @@ class LookerSource(DashboardServiceSource):
                     stackTrace=traceback.format_exc(),
                 )
             )
+
+    def close(self):
+        self.metadata.compute_percentile(Dashboard, self.today)
+        self.metadata.close()

metadata/ingestion/source/dashboard/metabase/client.py

@@ -13,7 +13,7 @@ REST Auth & Client for Metabase
 """
 import json
 import traceback
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import requests
 
@@ -23,6 +23,7 @@ from metadata.generated.schema.entity.services.connections.dashboard.metabaseCon
 from metadata.ingestion.connections.test_connections import SourceConnectionException
 from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.metabase.models import (
+    MetabaseChart,
     MetabaseCollection,
     MetabaseCollectionList,
     MetabaseDashboard,
@@ -32,7 +33,11 @@ from metadata.ingestion.source.dashboard.metabase.models import (
     MetabaseTable,
     MetabaseUser,
 )
-from metadata.utils.constants import AUTHORIZATION_HEADER, NO_ACCESS_TOKEN
+from metadata.utils.constants import (
+    AUTHORIZATION_HEADER,
+    DEFAULT_DASHBAORD,
+    NO_ACCESS_TOKEN,
+)
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -146,22 +151,76 @@ class MetabaseClient:
             logger.warning("Failed to fetch the collections list")
         return []
 
+    def get_charts_dict(self) -> Dict:
+        charts_dict = {}
+        try:
+            resp_charts = self.client.get("/card")
+            if resp_charts:
+                for chart_data in resp_charts:
+                    chart = MetabaseChart.model_validate(chart_data)
+                    charts_dict[chart.id] = chart
+            return charts_dict
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the cards")
+        return {}
+
+    def _create_default_dashboard_details(
+        self, orphan_charts_id: List
+    ) -> MetabaseDashboardDetails:
+        """
+        Returns:
+            MetabaseDashboardDetails object representing the default dashboard containing orphaned charts
+        """
+        return MetabaseDashboardDetails(
+            id=DEFAULT_DASHBAORD,
+            card_ids=orphan_charts_id,
+        )
+
+    def _process_dashboard_response(
+        self, resp_dashboard: Dict, charts_dict: Dict, dashboard_id: str
+    ) -> MetabaseDashboardDetails:
+        """
+        Process dashboard response and create MetabaseDashboardDetails object
+        """
+        if "ordered_cards" in resp_dashboard:
+            resp_dashboard["dashcards"] = resp_dashboard["ordered_cards"]
+
+        card_ids = []
+        for card in resp_dashboard.get("dashcards", []):
+            if card.get("card") and card["card"].get("id"):
+                card_id = str(card["card"]["id"])
+                card_ids.append(card_id)
+                if card_id in charts_dict:
+                    charts_dict[card_id].dashboard_ids.append(dashboard_id)
+
+        return MetabaseDashboardDetails(
+            description=resp_dashboard.get("description"),
+            name=resp_dashboard.get("name"),
+            id=resp_dashboard.get("id"),
+            creator_id=resp_dashboard.get("creator_id"),
+            collection_id=resp_dashboard.get("collection_id"),
+            card_ids=card_ids,
+        )
+
     def get_dashboard_details(
-        self, dashboard_id: str
+        self, dashboard_id: str, charts_dict: Dict, orphan_charts_id: List
     ) -> Optional[MetabaseDashboardDetails]:
         """
         Get Dashboard Details
         """
         if not dashboard_id:
             return None  # don't call api if dashboard_id is None
+        if dashboard_id == DEFAULT_DASHBAORD:
+            return self._create_default_dashboard_details(orphan_charts_id)
         try:
             resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}")
             if resp_dashboard:
                 # Small hack needed to support Metabase versions older than 0.48
                 # https://www.metabase.com/releases/metabase-48#fyi--breaking-changes
-                if "ordered_cards" in resp_dashboard:
-                    resp_dashboard["dashcards"] = resp_dashboard["ordered_cards"]
-                return MetabaseDashboardDetails(**resp_dashboard)
+                return self._process_dashboard_response(
+                    resp_dashboard, charts_dict, dashboard_id
+                )
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")

metadata/ingestion/source/dashboard/metabase/metadata.py

@@ -11,7 +11,7 @@
 """Metabase source module"""
 
 import traceback
-from typing import Any, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -20,6 +20,7 @@ from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboard import (
     Dashboard as LineageDashboard,
 )
+from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
     MetabaseConnection,
 )
@@ -42,9 +43,8 @@ from metadata.generated.schema.type.basic import (
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
+from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import search_table_entities
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.metabase.models import (
@@ -54,7 +54,9 @@ from metadata.ingestion.source.dashboard.metabase.models import (
     MetabaseDashboardDetails,
 )
 from metadata.utils import fqn
+from metadata.utils.constants import DEFAULT_DASHBAORD
 from metadata.utils.filters import filter_by_chart
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import (
     clean_uri,
     get_standard_chart_type,
@@ -92,16 +94,22 @@ class MetabaseSource(DashboardServiceSource):
     ):
         super().__init__(config, metadata)
         self.collections: List[MetabaseCollection] = []
+        self.dashboards_list: List[MetabaseDashboard] = []
+        self.charts_dict: Dict[str] = {}
+        self.orphan_charts_id: List[str] = []
+        self._default_dashboard_added = False
 
     def prepare(self):
         self.collections = self.client.get_collections_list()
+        self.charts_dict = self.client.get_charts_dict()
         return super().prepare()
 
     def get_dashboards_list(self) -> Optional[List[MetabaseDashboard]]:
         """
         Get List of all dashboards
         """
-        return self.client.get_dashboards_list(self.collections)
+        self.dashboards_list = self.client.get_dashboards_list(self.collections)
+        return self.dashboards_list
 
     def get_dashboard_name(self, dashboard: MetabaseDashboard) -> str:
         """
@@ -115,13 +123,39 @@ class MetabaseSource(DashboardServiceSource):
         """
         Get Dashboard Details
         """
-        return self.client.get_dashboard_details(dashboard.id)
+        retrieved_dashboards = self.client.get_dashboard_details(
+            dashboard.id, self.charts_dict, self.orphan_charts_id
+        )
+        if (
+            retrieved_dashboards
+            and dashboard == self.dashboards_list[-1]
+            and not self._default_dashboard_added
+        ):
+            # If processing the last dashboard, identify any orphaned charts (not associated with dashboards)
+            # and create a default dashboard to maintain visibility of these charts
+            self.orphan_charts_id = [
+                chart_id
+                for chart_id, chart in self.charts_dict.items()
+                if not chart.dashboard_ids
+            ]
+            if self.orphan_charts_id:
+                # add the default dashboard to the dashboards list
+                default_dashboard = MetabaseDashboard(
+                    name=DEFAULT_DASHBAORD,
+                    id=DEFAULT_DASHBAORD,
+                )
+                self.dashboards_list.append(default_dashboard)
+                self._default_dashboard_added = True
+        return retrieved_dashboards
 
     def get_project_name(self, dashboard_details: Any) -> Optional[str]:
         """
         Method to get the project name by searching the dataset using id in the workspace dict
         """
         try:
+            # Return default for the default dashboard containing orphaned charts
+            if dashboard_details.id == DEFAULT_DASHBAORD:
+                return DEFAULT_DASHBAORD
             if dashboard_details.collection_id:
                 collection_name = next(
                     (
@@ -164,10 +198,14 @@ class MetabaseSource(DashboardServiceSource):
         Method to Get Dashboard Entity
         """
         try:
+            # dashboard_url to be empty for default dashboard
             dashboard_url = (
-                f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-"
+                ""
+                if dashboard_details.id == DEFAULT_DASHBAORD
+                else f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-"
                 f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
             )
+
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(str(dashboard_details.id)),
                 sourceUrl=SourceUrl(dashboard_url),
@@ -211,10 +249,10 @@ class MetabaseSource(DashboardServiceSource):
         Returns:
             Iterable[CreateChartRequest]
         """
-        charts = dashboard_details.dashcards
-        for chart in charts:
+        chart_ids = dashboard_details.card_ids
+        for chart_id in chart_ids:
             try:
-                chart_details = chart.card
+                chart_details = self.charts_dict[chart_id]
                 if not chart_details.id or not chart_details.name:
                     continue
                 chart_url = (
@@ -240,7 +278,7 @@ class MetabaseSource(DashboardServiceSource):
                 yield Either(
                     left=StackTraceError(
                         name="Chart",
-                        error=f"Error creating chart [{chart}]: {exc}",
+                        error=f"Error creating chart [{self.charts_dict[chart_id]}]: {exc}",
                         stackTrace=traceback.format_exc(),
                     )
                 )
@@ -248,22 +286,20 @@ class MetabaseSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: MetabaseDashboardDetails,
-        db_service_name: Optional[str],
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method
 
         Args:
             dashboard_details
         """
-        if not db_service_name:
-            return
-        chart_list, dashboard_name = (
-            dashboard_details.dashcards,
+        chart_ids, dashboard_name = (
+            dashboard_details.card_ids,
             str(dashboard_details.id),
         )
-        for chart in chart_list:
+        for chart_id in chart_ids:
             try:
-                chart_details = chart.card
+                chart_details = self.charts_dict[chart_id]
                 if (
                     chart_details.dataset_query is None
                     or chart_details.dataset_query.type is None
@@ -296,11 +332,17 @@ class MetabaseSource(DashboardServiceSource):
                     )
                 )
 
-    def _get_database_service(self, db_service_name: str):
+    def _get_database_service(self, db_service_name: Optional[str]):
+        if not db_service_name:
+            return None
         return self.metadata.get_by_name(DatabaseService, db_service_name)
 
+    # pylint: disable=too-many-locals
     def _yield_lineage_from_query(
-        self, chart_details: MetabaseChart, db_service_name: str, dashboard_name: str
+        self,
+        chart_details: MetabaseChart,
+        db_service_name: Optional[str],
+        dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         database = self.client.get_database(chart_details.database_id)
 
@@ -323,20 +365,23 @@ class MetabaseSource(DashboardServiceSource):
             query,
             ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
             if db_service
-            else None,
+            else Dialect.ANSI,
         )
 
         for table in lineage_parser.source_tables:
             database_schema_name, table = fqn.split(str(table))[-2:]
             database_schema_name = self.check_database_schema_name(database_schema_name)
-            from_entities = search_table_entities(
-                metadata=self.metadata,
-                database=database_name,
-                service_name=db_service_name,
-                database_schema=database_schema_name,
-                table=table,
+            fqn_search_string = build_es_fqn_search_string(
+                database_name=database_name,
+                schema_name=database_schema_name,
+                service_name=db_service_name or "*",
+                table_name=table,
+            )
+            from_entities = self.metadata.search_in_any_service(
+                entity_type=Table,
+                fqn_search_string=fqn_search_string,
+                fetch_multiple_entities=True,
             )
-
             to_fqn = fqn.build(
                 self.metadata,
                 entity_type=LineageDashboard,
@@ -348,13 +393,16 @@ class MetabaseSource(DashboardServiceSource):
                 fqn=to_fqn,
             )
 
-            for from_entity in from_entities:
+            for from_entity in from_entities or []:
                 yield self._get_add_lineage_request(
                     to_entity=to_entity, from_entity=from_entity
                 )
 
     def _yield_lineage_from_api(
-        self, chart_details: MetabaseChart, db_service_name: str, dashboard_name: str
+        self,
+        chart_details: MetabaseChart,
+        db_service_name: Optional[str],
+        dashboard_name: str,
     ) -> Iterable[Either[AddLineageRequest]]:
         table = self.client.get_table(chart_details.table_id)
         table_name = table.name or table.display_name
@@ -363,14 +411,17 @@ class MetabaseSource(DashboardServiceSource):
             return
 
         database_name = table.db.details.db if table.db and table.db.details else None
-        from_entities = search_table_entities(
-            metadata=self.metadata,
-            database=database_name,
-            service_name=db_service_name,
-            database_schema=table.table_schema,
-            table=table_name,
+        fqn_search_string = build_es_fqn_search_string(
+            database_name=database_name,
+            schema_name=table.table_schema,
+            service_name=db_service_name or "*",
+            table_name=table_name,
+        )
+        from_entities = self.metadata.search_in_any_service(
+            entity_type=Table,
+            fqn_search_string=fqn_search_string,
+            fetch_multiple_entities=True,
         )
-
         to_fqn = fqn.build(
             self.metadata,
             entity_type=LineageDashboard,
@@ -383,7 +434,7 @@ class MetabaseSource(DashboardServiceSource):
             fqn=to_fqn,
         )
 
-        for from_entity in from_entities:
+        for from_entity in from_entities or []:
             yield self._get_add_lineage_request(
                 to_entity=to_entity, from_entity=from_entity
             )

metadata/ingestion/source/dashboard/metabase/models.py

@@ -82,6 +82,7 @@ class MetabaseChart(BaseModel):
     dataset_query: Optional[DatasetQuery] = None
     id: Optional[MetabaseStrId] = None
     display: Optional[str] = None
+    dashboard_ids: List[str] = []
 
 
 class DashCard(BaseModel):
@@ -94,7 +95,7 @@ class MetabaseDashboardDetails(BaseModel):
     """
 
     description: Optional[str] = None
-    dashcards: List[DashCard]
+    card_ids: List[str] = []
     name: Optional[str] = None
     id: MetabaseStrId
     creator_id: Optional[MetabaseStrId] = None

metadata/ingestion/source/dashboard/microstrategy/metadata.py

@@ -159,7 +159,9 @@ class MicrostrategySource(DashboardServiceSource):
                 )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: MstrDashboardDetails, db_service_name: str
+        self,
+        dashboard_details: MstrDashboardDetails,
+        db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Not Implemented"""
 

metadata/ingestion/source/dashboard/mode/metadata.py

@@ -20,6 +20,7 @@ from metadata.generated.schema.entity.data.chart import Chart, ChartType
 from metadata.generated.schema.entity.data.dashboard import (
     Dashboard as Lineage_Dashboard,
 )
+from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.dashboard.modeConnection import (
     ModeConnection,
 )
@@ -38,12 +39,12 @@ from metadata.generated.schema.type.basic import (
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.parser import LineageParser
-from metadata.ingestion.lineage.sql_lineage import search_table_entities
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.mode import client
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -130,7 +131,9 @@ class ModeSource(DashboardServiceSource):
         self.register_record(dashboard_request=dashboard_request)
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict, db_service_name: str
+        self,
+        dashboard_details: dict,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """Get lineage method"""
         try:
@@ -151,14 +154,18 @@ class ModeSource(DashboardServiceSource):
                     database_schema_name = self.check_database_schema_name(
                         database_schema_name
                     )
-                    from_entities = search_table_entities(
-                        metadata=self.metadata,
-                        database=data_source.get(client.DATABASE),
-                        service_name=db_service_name,
-                        database_schema=database_schema_name,
-                        table=table,
+                    fqn_search_string = build_es_fqn_search_string(
+                        database_name=data_source.get(client.DATABASE),
+                        schema_name=database_schema_name,
+                        service_name=db_service_name or "*",
+                        table_name=table,
+                    )
+                    from_entities = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
+                        fetch_multiple_entities=True,
                     )
-                    for from_entity in from_entities:
+                    for from_entity in from_entities or []:
                         to_entity = self.metadata.get_by_name(
                             entity=Lineage_Dashboard,
                             fqn=fqn.build(