openmetadata-ingestion 1.7.4.0__py3-none-any.whl → 1.7.4.1__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -54,7 +54,6 @@ from metadata.generated.schema.type.basic import (
 from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -240,6 +239,7 @@ class PowerbiSource(DashboardServiceSource):
             self.workspace_data.append(workspace)
             self.context.get().workspace = workspace
             self.filtered_dashboards = []
+            self.filtered_datamodels = []
             for dashboard in self.get_dashboards_list() or []:
                 dashboard_details = self.get_dashboard_details(dashboard)
                 dashboard_name = self.get_dashboard_name(dashboard_details)
@@ -532,6 +532,7 @@ class PowerbiSource(DashboardServiceSource):
                     ):
                         self.status.filter(dataset.name, "Data model filtered out.")
                         continue
+                    self.filtered_datamodels.append(dataset)
                     if isinstance(dataset, Dataset):
                         data_model_type = DataModelType.PowerBIDataModel.value
                         datamodel_columns = self._get_column_info(dataset)
@@ -606,7 +607,7 @@ class PowerbiSource(DashboardServiceSource):
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
-                    name="Lineage",
+                    name="Report and Dashboard Lineage",
                     error=f"Error to yield report and dashboard lineage details: {exc}",
                     stackTrace=traceback.format_exc(),
                 )
@@ -614,7 +615,6 @@ class PowerbiSource(DashboardServiceSource):
 
     def create_datamodel_report_lineage(
         self,
-        db_service_name: Optional[str],
         dashboard_details: PowerBIReport,
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """
@@ -631,45 +631,28 @@ class PowerbiSource(DashboardServiceSource):
                 entity=Dashboard,
                 fqn=report_fqn,
             )
+            datamodel_fqn = fqn.build(
+                self.metadata,
+                entity_type=DashboardDataModel,
+                service_name=self.context.get().dashboard_service,
+                data_model_name=dashboard_details.datasetId,
+            )
+            datamodel_entity = self.metadata.get_by_name(
+                entity=DashboardDataModel,
+                fqn=datamodel_fqn,
+            )
 
-            dataset = self._fetch_dataset_from_workspace(dashboard_details.datasetId)
-            if dataset:
-                datamodel_fqn = fqn.build(
-                    self.metadata,
-                    entity_type=DashboardDataModel,
-                    service_name=self.context.get().dashboard_service,
-                    data_model_name=dataset.id,
+            if datamodel_entity and report_entity:
+                yield self._get_add_lineage_request(
+                    to_entity=report_entity, from_entity=datamodel_entity
                 )
-                datamodel_entity = self.metadata.get_by_name(
-                    entity=DashboardDataModel,
-                    fqn=datamodel_fqn,
-                )
-
-                if datamodel_entity and report_entity:
-                    yield self._get_add_lineage_request(
-                        to_entity=report_entity, from_entity=datamodel_entity
-                    )
-
-                    for table in dataset.tables or []:
-                        yield self._get_table_and_datamodel_lineage(
-                            db_service_name=db_service_name,
-                            table=table,
-                            datamodel_entity=datamodel_entity,
-                        )
 
-                    # create the lineage between table and datamodel using the pbit files
-                    if self.client.file_client:
-                        yield from self.create_table_datamodel_lineage_from_files(
-                            db_service_name=db_service_name,
-                            datamodel_entity=datamodel_entity,
-                        )
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
-                    name=f"{db_service_name} Report Lineage",
+                    name=f"Datamodel and Report Lineage",
                     error=(
-                        "Error to yield datamodel and report lineage details for DB "
-                        f"service name [{db_service_name}]: {exc}"
+                        f"Error to yield datamodel and report lineage details: {exc}"
                     ),
                     stackTrace=traceback.format_exc(),
                 )
@@ -816,7 +799,7 @@ class PowerbiSource(DashboardServiceSource):
         db_service_name: Optional[str],
         table: PowerBiTable,
         datamodel_entity: DashboardDataModel,
-    ) -> Optional[Either[AddLineageRequest]]:
+    ) -> Iterable[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels
         """
@@ -837,13 +820,13 @@ class PowerbiSource(DashboardServiceSource):
                 column_lineage = self._get_column_lineage(
                     table_entity, datamodel_entity, columns_list
                 )
-                return self._get_add_lineage_request(
+                yield self._get_add_lineage_request(
                     to_entity=datamodel_entity,
                     from_entity=table_entity,
                     column_lineage=column_lineage,
                 )
         except Exception as exc:  # pylint: disable=broad-except
-            return Either(
+            yield Either(
                 left=StackTraceError(
                     name="DataModel Lineage for pbit files",
                     error=(
@@ -853,7 +836,6 @@ class PowerbiSource(DashboardServiceSource):
                     stackTrace=traceback.format_exc(),
                 )
             )
-        return None
 
     def create_table_datamodel_lineage_from_files(
         self,
@@ -882,7 +864,7 @@ class PowerbiSource(DashboardServiceSource):
 
             for datamodel_schema_file in datamodel_file_list:
                 for table in datamodel_schema_file.tables or []:
-                    yield self._get_table_and_datamodel_lineage(
+                    yield from self._get_table_and_datamodel_lineage(
                         db_service_name=db_service_name,
                         table=table,
                         datamodel_entity=datamodel_entity,
@@ -893,12 +875,153 @@ class PowerbiSource(DashboardServiceSource):
                     name="DataModel Lineage",
                     error=(
                         "Error to yield datamodel lineage details for DB "
-                        f"service name [{db_service_name}]: {exc}"
+                        f"service name [{str(db_service_name)}]: {exc}"
                     ),
                     stackTrace=traceback.format_exc(),
                 )
             )
 
+    def create_dataset_upstream_dataflow_lineage(
+        self, datamodel: Dataset, datamodel_entity: DashboardDataModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Create lineage between dataset and upstreamDataflow
+        """
+        for upstream_dataflow in datamodel.upstreamDataflows or []:
+            try:
+                if not upstream_dataflow.targetDataflowId:
+                    logger.debug(
+                        f"No targetDataflowId found for upstreamDataflow in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataflow"
+                    )
+                    continue
+                upstream_dataflow_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataflow.targetDataflowId,
+                )
+                upstream_dataflow_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataflow_fqn,
+                )
+                if upstream_dataflow_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataflow_entity,
+                        to_entity=datamodel_entity,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataflow entity with id={str(upstream_dataflow.targetDataflowId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataset and UpstreamDataflow Lineage",
+                        error=(
+                            "Error to yield dataset and upstreamDataflow lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataflow.targetDataflowId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def create_dataset_upstream_dataset_lineage(
+        self, datamodel: Dataset, datamodel_entity: DashboardDataModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Create lineage between dataset and upstreamDataset
+        """
+        for upstream_dataset in datamodel.upstreamDatasets or []:
+            try:
+                if not upstream_dataset.targetDatasetId:
+                    logger.debug(
+                        f"No targetDatasetId found for upstreamDataset in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataset"
+                    )
+                    continue
+                upstream_dataset_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataset.targetDatasetId,
+                )
+                upstream_dataset_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataset_fqn,
+                )
+                if upstream_dataset_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataset_entity,
+                        to_entity=datamodel_entity,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataset entity with id={str(upstream_dataset.targetDatasetId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataset and UpstreamDataset Lineage",
+                        error=(
+                            "Error to yield dataset and upstreamDataset lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataset.targetDatasetId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def create_dataflow_upstream_dataflow_lineage(
+        self, datamodel: Dataflow, datamodel_entity: DashboardDataModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Create lineage between dataflow and upstreamDataflow
+        """
+        for upstream_dataflow in datamodel.upstreamDataflows or []:
+            try:
+                if not upstream_dataflow.targetDataflowId:
+                    logger.debug(
+                        f"No targetDataflowId found for upstreamDataflow in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataflow"
+                    )
+                    continue
+                upstream_dataflow_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataflow.targetDataflowId,
+                )
+                upstream_dataflow_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataflow_fqn,
+                )
+                if upstream_dataflow_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataflow_entity,
+                        to_entity=datamodel_entity,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataflow entity with id={str(upstream_dataflow.targetDataflowId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataflow and UpstreamDataflow Lineage",
+                        error=(
+                            f"Error to yield dataflow and upstreamDataflow lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataflow.targetDataflowId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: Group,
@@ -913,7 +1036,6 @@ class PowerbiSource(DashboardServiceSource):
             try:
                 if isinstance(dashboard_details, PowerBIReport):
                     yield from self.create_datamodel_report_lineage(
-                        db_service_name=db_service_name,
                         dashboard_details=dashboard_details,
                     )
                 if isinstance(dashboard_details, PowerBIDashboard):
@@ -924,27 +1046,83 @@ class PowerbiSource(DashboardServiceSource):
                 yield Either(
                     left=StackTraceError(
                         name="Dashboard Lineage",
-                        error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
+                        error=f"Error to yield dashboard lineage details for DB service name [{str(db_service_name)}]: {exc}",
                         stackTrace=traceback.format_exc(),
                     )
                 )
-
-    def yield_dashboard_lineage(
-        self, dashboard_details: Group
-    ) -> Iterable[Either[OMetaLineageRequest]]:
         """
-        Yields lineage if config is enabled.
+        Iterate loop for filtered datamodels so datamodels which are not connected to 
+        any report but have tables would be eligible for a dataset-db_table lineage.
+        Also create below lineages: 
+        1. dataset-db_table
+        2. dataset-upstreamDataflow
+        3. dataset-upstreamDataset
+        4. dataflow-upstreamDataflow
+        """
+        for datamodel in self.filtered_datamodels or []:
+            try:
+                datamodel_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=datamodel.id,
+                )
+                datamodel_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=datamodel_fqn,
+                )
+                if datamodel_entity:
+                    if isinstance(datamodel, Dataset):
+                        # 1. datamodel-db_table lineage
+                        for table in datamodel.tables or []:
+                            yield from self._get_table_and_datamodel_lineage(
+                                db_service_name=db_service_name,
+                                table=table,
+                                datamodel_entity=datamodel_entity,
+                            )
+                        # 2. dataset-upstreamDataflow lineage
+                        yield from self.create_dataset_upstream_dataflow_lineage(
+                            datamodel, datamodel_entity
+                        )
+                        # 3. dataset-upstreamDataset lineage
+                        yield from self.create_dataset_upstream_dataset_lineage(
+                            datamodel, datamodel_entity
+                        )
+                        # create the lineage between table and datamodel using the pbit files
+                        if self.client.file_client:
+                            yield from self.create_table_datamodel_lineage_from_files(
+                                db_service_name=db_service_name,
+                                datamodel_entity=datamodel_entity,
+                            )
+                    elif isinstance(datamodel, Dataflow):
+                        # create dataflow-upstreamDataflow lineage
+                        yield from self.create_dataflow_upstream_dataflow_lineage(
+                            datamodel, datamodel_entity
+                        )
+                    else:
+                        logger.warning(
+                            f"Unknown datamodel type: {type(datamodel)}, name: {datamodel.name}"
+                        )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Datamodel Lineage",
+                        error=f"Error to yield datamodel lineage details for DB service name [{str(db_service_name)}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
 
-        We will look for the data in all the services
-        we have informed.
+    def yield_datamodel_dashboard_lineage(
+        self,
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Returns:
+            Lineage request between Data Models and Dashboards
+        """
+        """
+            We're implementing this differently inside `yield_dashboard_lineage_details`
+            since we have report and dashboard both as dashboard.
         """
-        db_service_names = self.get_db_service_names()
-        if not db_service_names:
-            yield from self.yield_dashboard_lineage_details(dashboard_details) or []
-        for db_service_name in db_service_names or []:
-            yield from self.yield_dashboard_lineage_details(
-                dashboard_details, db_service_name
-            ) or []
 
     def _fetch_dataset_from_workspace(
         self, dataset_id: Optional[str]

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -174,6 +174,16 @@ class DatasetExpression(BaseModel):
     expression: Optional[str] = None
 
 
+class UpstreaDataflow(BaseModel):
+    groupId: Optional[str] = None
+    targetDataflowId: Optional[str] = None
+
+
+class UpstreaDataset(BaseModel):
+    groupId: Optional[str] = None
+    targetDatasetId: Optional[str] = None
+
+
 class Dataset(BaseModel):
     """
     PowerBI Dataset Model
@@ -187,6 +197,8 @@ class Dataset(BaseModel):
     users: Optional[List[PowerBIUser]] = []
     expressions: Optional[List[DatasetExpression]] = []
     configuredBy: Optional[str] = None
+    upstreamDataflows: Optional[List[UpstreaDataflow]] = []
+    upstreamDatasets: Optional[List[UpstreaDataset]] = []
 
 
 class DatasetResponse(BaseModel):
@@ -205,6 +217,7 @@ class Dataflow(BaseModel):
     description: Optional[str] = None
     users: Optional[List[PowerBIUser]] = []
     modifiedBy: Optional[str] = None
+    upstreamDataflows: Optional[List[UpstreaDataflow]] = []
 
 
 class Group(BaseModel):

metadata/ingestion/source/dashboard/tableau/client.py

@@ -20,6 +20,7 @@ from cached_property import cached_property
 from tableauserverclient import (
     Pager,
     PersonalAccessTokenAuth,
+    ProjectItem,
     Server,
     TableauAuth,
     ViewItem,
@@ -89,6 +90,7 @@ class TableauClient:
         self.pagination_limit = pagination_limit
         self.custom_sql_table_queries: Dict[str, List[str]] = {}
         self.owner_cache: Dict[str, TableauOwner] = {}
+        self.all_projects: List[ProjectItem] = []
 
     @cached_property
     def server_info(self) -> Callable:
@@ -148,10 +150,60 @@ class TableauClient:
 
         return charts, view_count
 
+    def get_all_projects(self) -> None:
+        """
+        Get all projects from the tableau server
+        """
+        try:
+            logger.debug("Getting all projects from the tableau server")
+            all_projects: List[ProjectItem] = []
+            for project in Pager(self.tableau_server.projects):
+                all_projects.append(project)
+            self.all_projects = all_projects
+        except Exception as e:
+            logger.debug(f"Failed to get all projects: {str(e)}")
+
+    def get_project_parents_by_id(self, project_id: str) -> Optional[str]:
+        """
+        Get the parents of a project by id
+        """
+        try:
+            parent_projects = []
+            current_project_id = project_id
+
+            while current_project_id:
+                # Find project with current ID
+                project = next(
+                    (
+                        proj
+                        for proj in self.all_projects
+                        if str(proj.id) == str(current_project_id)
+                    ),
+                    None,
+                )
+
+                if not project:
+                    break
+
+                parent_projects.append(project.name)
+
+                # Get parent ID and continue loop if exists
+                current_project_id = (
+                    project.parent_id if hasattr(project, "parent_id") else None
+                )
+
+            if parent_projects:
+                parent_projects = ".".join(reversed(parent_projects))
+                return parent_projects
+        except Exception as e:
+            logger.debug(f"Failed to get project parents by id: {str(e)}")
+        return None
+
     def get_workbooks(self) -> Iterable[TableauDashboard]:
         """
         Fetch all tableau workbooks
         """
+        self.get_all_projects()
         self.cache_custom_sql_tables()
         for workbook in Pager(self.tableau_server.workbooks):
             try:
@@ -322,7 +374,7 @@ class TableauClient:
         if datasource_id in self.custom_sql_table_queries:
             logger.debug(f"Found cached queries for datasource {datasource_id}")
             return self.custom_sql_table_queries[datasource_id]
-
+        logger.debug(f"No cached queries for datasource {datasource_id}")
         return None
 
     def cache_custom_sql_tables(self) -> None:

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -641,6 +641,10 @@ class TableauSource(DashboardServiceSource):
                                     fqn_search_string=fqn_search_string,
                                     fetch_multiple_entities=True,
                                 )
+                                if not from_entities:
+                                    logger.debug(
+                                        f"No table entities found for custom SQL lineage. fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
+                                    )
                                 for table_entity in from_entities:
                                     yield self._get_add_lineage_request(
                                         to_entity=upstream_data_model_entity,
@@ -811,6 +815,9 @@ class TableauSource(DashboardServiceSource):
             )
             if table_entity:
                 return [TableAndQuery(table=table_entity)]
+            logger.debug(
+                f"No table entity found for lineage using GraphQL APIs. fqn_search_string={fqn_search_string}, table_name={table_name}"
+            )
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Error to get tables for lineage using GraphQL Apis: {exc}")
@@ -864,6 +871,10 @@ class TableauSource(DashboardServiceSource):
                         fqn_search_string=fqn_search_string,
                         fetch_multiple_entities=True,
                     )
+                    if not from_entities:
+                        logger.debug(
+                            f"No table entities found for lineage using SQL Queries. fqn_search_string={fqn_search_string}, table_name={table_name}, query={custom_sql_table.query}"
+                        )
                     tables_list.extend(
                         [
                             TableAndQuery(table=table, query=custom_sql_table.query)
@@ -980,6 +991,19 @@ class TableauSource(DashboardServiceSource):
             )
         return None
 
+    def get_project_names(self, dashboard_details: Any) -> Optional[str]:
+        """
+        Get the project / workspace / folder / collection names of the dashboard
+        """
+        try:
+            return self.client.get_project_parents_by_id(dashboard_details.project.id)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error fetching project names for {dashboard_details.id}: {exc}"
+            )
+        return None
+
     def yield_dashboard_usage(
         self, dashboard_details: TableauDashboard
     ) -> Iterable[Either[DashboardUsage]]:

metadata/ingestion/source/database/bigquery/profiler/system.py

@@ -154,7 +154,7 @@ class BigQuerySystemMetricsSource(
                     "rowsAffected": getattr(q, rows_affected_field),
                 }
                 for q in query_results
-                if getattr(q, rows_affected_field) > 0
+                if getattr(q, rows_affected_field, -1) > 0
                 and q.project_id == project_id
                 and q.dataset_id == dataset_id
                 and q.table_name == table

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -461,6 +461,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     )
                     logger.error(error)
                     self.status.failed_profiler(error, traceback.format_exc())
+                    break
 
         # If we've exhausted all retries without success, return a tuple of None values
         return None, None, None