openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -10,9 +10,13 @@
 #  limitations under the License.
 """PowerBI source module"""
 
+import re
 import traceback
 from typing import Any, Iterable, List, Optional, Union
 
+from pydantic import EmailStr
+from pydantic_core import PydanticCustomError
+
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.data.createDashboardDataModel import (
@@ -47,17 +51,27 @@ from metadata.generated.schema.type.basic import (
     Markdown,
     SourceUrl,
 )
+from metadata.generated.schema.type.entityLineage import ColumnLineage
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.lineage.models import Dialect
+from metadata.ingestion.lineage.parser import LineageParser
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
+from metadata.ingestion.source.dashboard.powerbi.databricks_parser import (
+    parse_databricks_native_query_source,
+)
 from metadata.ingestion.source.dashboard.powerbi.models import (
+    Dataflow,
     Dataset,
     Group,
     PowerBIDashboard,
+    PowerBiMeasureModel,
     PowerBIReport,
     PowerBiTable,
+    ReportPage,
 )
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils import fqn
@@ -65,13 +79,17 @@ from metadata.utils.filters import (
     filter_by_chart,
     filter_by_dashboard,
     filter_by_datamodel,
-    filter_by_project,
 )
+from metadata.utils.fqn import build_es_fqn_search_string
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+OWNER_ACCESS_RIGHTS_KEYWORDS = ["owner", "write", "admin"]
+SNOWFLAKE_QUERY_EXPRESSION_KW = "Value.NativeQuery(Snowflake.Databases("
+DATABRICKS_QUERY_EXPRESSION_KW = "Value.NativeQuery(Databricks.Catalogs("
+
 
 class PowerbiSource(DashboardServiceSource):
     """PowerBi Source Class"""
@@ -91,31 +109,111 @@ class PowerbiSource(DashboardServiceSource):
         self.workspace_data = []
         self.datamodel_file_mappings = []
 
-    def prepare(self):
-        return super().prepare()
-
     def close(self):
         self.metadata.close()
         if self.client.file_client:
             self.client.file_client.delete_tmp_files()
 
-    def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
+    def get_org_workspace_data(self) -> Iterable[Optional[Group]]:
         """
-        Method to get the workspaces filtered by project filter pattern
+        fetch all the workspace data for non-admin users
         """
-        filtered_groups = []
-        for group in groups:
-            if filter_by_project(
-                self.source_config.projectFilterPattern,
-                group.name,
-            ):
-                self.status.filter(
-                    group.name,
-                    "Workspace Filtered Out",
+        filter_pattern = self.source_config.projectFilterPattern
+        workspaces = self.client.api_client.fetch_all_workspaces(filter_pattern)
+        for workspace in workspaces:
+            # add the dashboards to the workspace
+            workspace.dashboards.extend(
+                self.client.api_client.fetch_all_org_dashboards(group_id=workspace.id)
+                or []
+            )
+            for dashboard in workspace.dashboards:
+                # add the tiles to the dashboards
+                dashboard.tiles.extend(
+                    self.client.api_client.fetch_all_org_tiles(
+                        group_id=workspace.id, dashboard_id=dashboard.id
+                    )
+                    or []
+                )
+
+            # add the reports to the workspaces
+            workspace.reports.extend(
+                self.client.api_client.fetch_all_org_reports(group_id=workspace.id)
+                or []
+            )
+
+            # add the datasets to the workspaces
+            workspace.datasets.extend(
+                self.client.api_client.fetch_all_org_datasets(group_id=workspace.id)
+                or []
+            )
+            for dataset in workspace.datasets:
+                # add the tables to the datasets
+                dataset.tables.extend(
+                    self.client.api_client.fetch_dataset_tables(
+                        group_id=workspace.id, dataset_id=dataset.id
+                    )
+                    or []
                 )
-                continue
-            filtered_groups.append(group)
-        return filtered_groups
+            yield workspace
+
+    def get_admin_workspace_data(self) -> Iterable[Optional[Group]]:
+        """
+        fetch all the workspace data
+        """
+        filter_pattern = self.source_config.projectFilterPattern
+        workspaces = self.client.api_client.fetch_all_workspaces(filter_pattern)
+        if workspaces:
+            workspace_id_list = [workspace.id for workspace in workspaces]
+
+            # Start the scan of the available workspaces for dashboard metadata
+            workspace_paginated_list = [
+                workspace_id_list[i : i + self.pagination_entity_per_page]
+                for i in range(
+                    0, len(workspace_id_list), self.pagination_entity_per_page
+                )
+            ]
+            count = 1
+            for workspace_ids_chunk in workspace_paginated_list:
+                logger.info(
+                    f"Scanning {count}/{len(workspace_paginated_list)} set of workspaces"
+                )
+                workspace_scan = self.client.api_client.initiate_workspace_scan(
+                    workspace_ids_chunk
+                )
+                if not workspace_scan:
+                    logger.error(
+                        f"Error initiating workspace scan for ids:{str(workspace_ids_chunk)}\n moving to next set of workspaces"
+                    )
+                    count += 1
+                    continue
+
+                # Keep polling the scan status endpoint to check if scan is succeeded
+                workspace_scan_status = self.client.api_client.wait_for_scan_complete(
+                    scan_id=workspace_scan.id
+                )
+                if not workspace_scan_status:
+                    logger.error(
+                        f"Max poll hit to scan status for scan_id: {workspace_scan.id}, moving to next set of workspaces"
+                    )
+                    count += 1
+                    continue
+
+                # Get scan result for successfull scan
+                response = self.client.api_client.fetch_workspace_scan_result(
+                    scan_id=workspace_scan.id
+                )
+                if not response:
+                    logger.error(
+                        f"Error getting workspace scan result for scan_id: {workspace_scan.id}"
+                    )
+                    count += 1
+                    continue
+                for active_workspace in response.workspaces:
+                    if active_workspace.state == "Active":
+                        yield active_workspace
+                count += 1
+        else:
+            logger.error("Unable to fetch any PowerBI workspaces")
 
     @classmethod
     def create(
@@ -129,62 +227,31 @@ class PowerbiSource(DashboardServiceSource):
             )
         return cls(config, metadata)
 
-    def prepare_workspace_data(self, workspace: Group):
-        """prepare one workspace data at a time"""
-        # add the dashboards to the groups
-        workspace.dashboards.extend(
-            self.client.api_client.fetch_all_org_dashboards(group_id=workspace.id) or []
-        )
-        for dashboard in workspace.dashboards:
-            # add the tiles to the dashboards
-            dashboard.tiles.extend(
-                self.client.api_client.fetch_all_org_tiles(
-                    group_id=workspace.id, dashboard_id=dashboard.id
-                )
-                or []
-            )
-
-        # add the reports to the groups
-        workspace.reports.extend(
-            self.client.api_client.fetch_all_org_reports(group_id=workspace.id) or []
-        )
-
-        # add the datasets to the groups
-        workspace.datasets.extend(
-            self.client.api_client.fetch_all_org_datasets(group_id=workspace.id) or []
-        )
-        for dataset in workspace.datasets:
-            # add the tables to the datasets
-            dataset.tables.extend(
-                self.client.api_client.fetch_dataset_tables(
-                    group_id=workspace.id, dataset_id=dataset.id
-                )
-                or []
-            )
+    def _prepare_workspace_data(self):
+        """
+        - Since we get all the required info i.e. reports, dashboards, charts, datasets
+          with workflow scan approach, we are populating bulk data for workspace.
+        - Some individual APIs are not able to yield data with details.
+        """
+        if self.service_connection.useAdminApis:
+            for workspace in self.get_admin_workspace_data():
+                yield workspace
+        else:
+            for workspace in self.get_org_workspace_data():
+                yield workspace
 
     def get_dashboard(self) -> Any:
         """
         Method to iterate through dashboard lists filter dashboards & yield dashboard details
         """
-        # fetch all workspaces/groups & apply filter pattern
-        all_workspaces = self.client.api_client.fetch_all_workspaces() or []
-        all_workspaces = self.get_filtered_workspaces(all_workspaces)
-        for workspace in all_workspaces:
-            # prepare additional data for specific workspace (datasets, reports, dashboards)
-            self.prepare_workspace_data(workspace)
-            self.context.get().workspace = workspace
+        for workspace in self._prepare_workspace_data():
             self.workspace_data.append(workspace)
-            for dashboard in self.get_dashboards_list():
-                try:
-                    dashboard_details = self.get_dashboard_details(dashboard)
-                except Exception as exc:
-                    logger.debug(traceback.format_exc())
-                    logger.warning(
-                        f"Cannot extract dashboard details from {dashboard}: {exc}"
-                    )
-                    continue
+            self.context.get().workspace = workspace
+            self.filtered_dashboards = []
+            self.filtered_datamodels = []
+            for dashboard in self.get_dashboards_list() or []:
+                dashboard_details = self.get_dashboard_details(dashboard)
                 dashboard_name = self.get_dashboard_name(dashboard_details)
-
                 if filter_by_dashboard(
                     self.source_config.dashboardFilterPattern,
                     dashboard_name,
@@ -194,7 +261,8 @@ class PowerbiSource(DashboardServiceSource):
                         "Dashboard Filtered Out",
                     )
                     continue
-                yield dashboard_details
+                self.filtered_dashboards.append(dashboard_details)
+            yield workspace
 
     def get_dashboards_list(
         self,
@@ -231,16 +299,48 @@ class PowerbiSource(DashboardServiceSource):
         """
         return (
             f"{clean_uri(self.service_connection.hostPort)}/groups/"
-            f"{workspace_id}/dashboards/{dashboard_id}"
+            f"{workspace_id}/dashboards/{dashboard_id}?experience=power-bi"
         )
 
     def _get_report_url(self, workspace_id: str, dashboard_id: str) -> str:
         """
         Method to build the dashboard url
         """
+        page_id = ""
+        try:
+            pages: Optional[
+                List[ReportPage]
+            ] = self.client.api_client.fetch_report_pages(workspace_id, dashboard_id)
+            if len(pages) >= 1:
+                # get first page out of multiple pages otherwise
+                # get page if of single page
+                page_id = pages[0].name
+            page_id = f"/{page_id}" if page_id else ""
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error building report page url: {exc}")
+        # https://app.powerbi.com/groups/4e57dcbb-***/reports/a2902011-***/098b***?experience=power-bi
+        return (
+            f"{clean_uri(self.service_connection.hostPort)}/groups/"
+            f"{workspace_id}/reports/{dashboard_id}{page_id}?experience=power-bi"
+        )
+
+    def _get_dataset_url(self, workspace_id: str, dataset_id: str) -> str:
+        """
+        Method to build the dataset url
+        """
         return (
             f"{clean_uri(self.service_connection.hostPort)}/groups/"
-            f"{workspace_id}/reports/{dashboard_id}"
+            f"{workspace_id}/datasets/{dataset_id}/details?experience=power-bi"
+        )
+
+    def _get_dataflow_url(self, workspace_id: str, dataflow_id: str) -> str:
+        """
+        Method to build the dataset url
+        """
+        return (
+            f"{clean_uri(self.service_connection.hostPort)}/groups/"
+            f"{workspace_id}/dataflows/{dataflow_id}?experience=power-bi"
         )
 
     def _get_chart_url(
@@ -257,45 +357,147 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/{chart_url_postfix}"
         )
 
-    def yield_datamodel(
-        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
+    def yield_dashboard(
+        self, dashboard_details: Group
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """
-        Method to yield datamodel for each workspace
+        Method to Get Dashboard Entity, Dashboard Charts & Lineage
         """
-        workspace_datasets = self.context.get().workspace.datasets
-        for dataset in workspace_datasets:
-            if filter_by_datamodel(
-                self.source_config.dataModelFilterPattern, dataset.name
-            ):
-                self.status.filter(dataset.name, "Data model filtered out.")
-                continue
-            try:
-                data_model_request = CreateDashboardDataModelRequest(
-                    name=EntityName(dataset.id),
-                    displayName=dataset.name,
-                    description=Markdown(dataset.description)
-                    if dataset.description
-                    else None,
-                    service=FullyQualifiedEntityName(
-                        self.context.get().dashboard_service
-                    ),
-                    dataModelType=DataModelType.PowerBIDataModel.value,
-                    serviceType=DashboardServiceType.PowerBI.value,
-                    columns=self._get_column_info(dataset),
-                    project=self.get_project_name(dashboard_details),
+        try:
+            for dashboard in self.filtered_dashboards or []:
+                dashboard_details = self.get_dashboard_details(dashboard)
+                if isinstance(dashboard_details, PowerBIDashboard):
+                    dashboard_request = CreateDashboardRequest(
+                        name=EntityName(dashboard_details.id),
+                        sourceUrl=SourceUrl(
+                            self._get_dashboard_url(
+                                workspace_id=self.context.get().workspace.id,
+                                dashboard_id=dashboard_details.id,
+                            )
+                        ),
+                        project=self.get_project_name(dashboard_details),
+                        displayName=dashboard_details.displayName,
+                        dashboardType=DashboardType.Dashboard,
+                        charts=[
+                            FullyQualifiedEntityName(
+                                fqn.build(
+                                    self.metadata,
+                                    entity_type=Chart,
+                                    service_name=self.context.get().dashboard_service,
+                                    chart_name=chart,
+                                )
+                            )
+                            for chart in self.context.get().charts or []
+                        ],
+                        service=FullyQualifiedEntityName(
+                            self.context.get().dashboard_service
+                        ),
+                        owners=self.get_owner_ref(dashboard_details=dashboard_details),
+                    )
+                else:
+                    dashboard_request = CreateDashboardRequest(
+                        name=EntityName(dashboard_details.id),
+                        dashboardType=DashboardType.Report,
+                        sourceUrl=SourceUrl(
+                            self._get_report_url(
+                                workspace_id=self.context.get().workspace.id,
+                                dashboard_id=dashboard_details.id,
+                            )
+                        ),
+                        project=self.get_project_name(dashboard_details),
+                        displayName=dashboard_details.name,
+                        service=self.context.get().dashboard_service,
+                        owners=self.get_owner_ref(dashboard_details=dashboard_details),
+                    )
+                yield Either(right=dashboard_request)
+                self.register_record(dashboard_request=dashboard_request)
+        except Exception as exc:  # pylint: disable=broad-except
+            yield Either(
+                left=StackTraceError(
+                    name=dashboard_details.name,
+                    error=f"Error creating dashboard [{dashboard_details}]: {exc}",
+                    stackTrace=traceback.format_exc(),
                 )
-                yield Either(right=data_model_request)
-                self.register_record_datamodel(datamodel_request=data_model_request)
+            )
 
-            except Exception as exc:
-                yield Either(
-                    left=StackTraceError(
-                        name=dataset.name,
-                        error=f"Error yielding Data Model [{dataset.name}]: {exc}",
-                        stackTrace=traceback.format_exc(),
-                    )
+    def yield_dashboard_chart(
+        self, dashboard_details: Group
+    ) -> Iterable[Either[CreateChartRequest]]:
+        """Get chart method
+        Args:
+            dashboard_details:
+        Returns:
+            Iterable[Chart]
+        """
+        for dashboard in self.filtered_dashboards or []:
+            dashboard_details = self.get_dashboard_details(dashboard)
+            if isinstance(dashboard_details, PowerBIDashboard):
+                charts = dashboard_details.tiles
+                for chart in charts or []:
+                    try:
+                        chart_title = chart.title
+                        chart_display_name = chart_title if chart_title else chart.id
+                        if filter_by_chart(
+                            self.source_config.chartFilterPattern, chart_display_name
+                        ):
+                            self.status.filter(
+                                chart_display_name, "Chart Pattern not Allowed"
+                            )
+                            continue
+                        yield Either(
+                            right=CreateChartRequest(
+                                name=EntityName(chart.id),
+                                displayName=chart_display_name,
+                                chartType=ChartType.Other.value,
+                                sourceUrl=SourceUrl(
+                                    self._get_chart_url(
+                                        report_id=chart.reportId,
+                                        workspace_id=self.context.get().workspace.id,
+                                        dashboard_id=dashboard_details.id,
+                                    )
+                                ),
+                                service=FullyQualifiedEntityName(
+                                    self.context.get().dashboard_service
+                                ),
+                            )
+                        )
+                    except Exception as exc:
+                        yield Either(
+                            left=StackTraceError(
+                                name=chart.title,
+                                error=f"Error creating chart [{chart.title}]: {exc}",
+                                stackTrace=traceback.format_exc(),
+                            )
+                        )
+
+    def _get_child_measures(self, table: PowerBiTable) -> List[Column]:
+        """
+        Extract the measures of the table
+        """
+        measures = []
+        for measure in table.measures or []:
+            try:
+                measure_type = DataType.MEASURE_VISIBLE
+                if measure.isHidden:
+                    measure_type = DataType.MEASURE_HIDDEN
+                expression_text = f"Expression : {measure.expression}"
+                description_text = (
+                    f"Description : {measure.description}"
+                    if measure.description
+                    else ""
+                )
+                description_field_text = f"{expression_text}\n\n{description_text}"
+                parsed_measure = PowerBiMeasureModel(
+                    dataType=measure_type,
+                    dataTypeDisplay=measure_type,
+                    name=measure.name,
+                    description=description_field_text,
                 )
+                measures.append(Column(**parsed_measure.model_dump()))
+            except Exception as err:
+                logger.debug(traceback.format_exc())
+                logger.debug(f"Error processing datamodel nested measure: {err}")
+        return measures
 
     def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
         """
@@ -305,14 +507,15 @@ class PowerbiSource(DashboardServiceSource):
         for column in table.columns or []:
             try:
                 parsed_column = {
-                    "dataTypeDisplay": column.dataType
-                    if column.dataType
-                    else DataType.UNKNOWN.value,
+                    "dataTypeDisplay": (
+                        column.dataType if column.dataType else DataType.UNKNOWN.value
+                    ),
                     "dataType": ColumnTypeParser.get_column_type(
                         column.dataType if column.dataType else None
                     ),
                     "name": column.name,
                     "displayName": column.name,
+                    "description": column.description,
                 }
                 if column.dataType and column.dataType == DataType.ARRAY.value:
                     parsed_column["arrayDataType"] = DataType.UNKNOWN
@@ -327,79 +530,103 @@ class PowerbiSource(DashboardServiceSource):
         datasource_columns = []
         for table in dataset.tables or []:
             try:
+                table_display_name = None
+                if self.service_connection.displayTableNameFromSource:
+                    table_display_name = self.parse_table_name_from_source(table=table)
+                    if table_display_name:
+                        logger.debug(
+                            f"Parsed Table display name: {table_display_name} for table: {table.name}"
+                        )
+                if not table_display_name:
+                    table_display_name = table.name
                 parsed_table = {
                     "dataTypeDisplay": "PowerBI Table",
                     "dataType": DataType.TABLE,
                     "name": table.name,
-                    "displayName": table.name,
+                    "displayName": table_display_name,
                     "description": table.description,
+                    "children": [],
                 }
                 child_columns = self._get_child_columns(table=table)
+                child_measures = self._get_child_measures(table=table)
                 if child_columns:
                     parsed_table["children"] = child_columns
+                if child_measures:
+                    parsed_table["children"].extend(child_measures)
                 datasource_columns.append(Column(**parsed_table))
             except Exception as exc:
                 logger.debug(traceback.format_exc())
                 logger.warning(f"Error to yield datamodel column: {exc}")
         return datasource_columns
 
-    def yield_dashboard(
-        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
-    ) -> Iterable[Either[CreateDashboardRequest]]:
+    def _get_datamodels_list(self) -> List[Union[Dataset, Dataflow]]:
         """
-        Method to Get Dashboard Entity, Dashboard Charts & Lineage
+        Get All the Powerbi Datasets
+        """
+        return (
+            self.context.get().workspace.datasets
+            + self.context.get().workspace.dataflows
+        )
+
+    def yield_datamodel(
+        self, dashboard_details: Group
+    ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
+        """
+        Get All the Powerbi Datasets
         """
         try:
-            if isinstance(dashboard_details, PowerBIDashboard):
-                dashboard_request = CreateDashboardRequest(
-                    name=EntityName(dashboard_details.id),
-                    sourceUrl=SourceUrl(
-                        self._get_dashboard_url(
+            if self.source_config.includeDataModels:
+                for dataset in self._get_datamodels_list() or []:
+                    if filter_by_datamodel(
+                        self.source_config.dataModelFilterPattern, dataset.name
+                    ):
+                        self.status.filter(dataset.name, "Data model filtered out.")
+                        continue
+                    self.filtered_datamodels.append(dataset)
+                    if isinstance(dataset, Dataset):
+                        data_model_type = DataModelType.PowerBIDataModel.value
+                        datamodel_columns = self._get_column_info(dataset)
+                        source_url = self._get_dataset_url(
                             workspace_id=self.context.get().workspace.id,
-                            dashboard_id=dashboard_details.id,
+                            dataset_id=dataset.id,
                         )
-                    ),
-                    project=self.get_project_name(dashboard_details=dashboard_details),
-                    displayName=dashboard_details.displayName,
-                    dashboardType=DashboardType.Dashboard,
-                    charts=[
-                        FullyQualifiedEntityName(
-                            fqn.build(
-                                self.metadata,
-                                entity_type=Chart,
-                                service_name=self.context.get().dashboard_service,
-                                chart_name=chart,
-                            )
-                        )
-                        for chart in self.context.get().charts or []
-                    ],
-                    service=FullyQualifiedEntityName(
-                        self.context.get().dashboard_service
-                    ),
-                    owners=self.get_owner_ref(dashboard_details=dashboard_details),
-                )
-            else:
-                dashboard_request = CreateDashboardRequest(
-                    name=EntityName(dashboard_details.id),
-                    dashboardType=DashboardType.Report,
-                    sourceUrl=SourceUrl(
-                        self._get_report_url(
+                    elif isinstance(dataset, Dataflow):
+                        data_model_type = DataModelType.PowerBIDataFlow.value
+                        datamodel_columns = []
+                        source_url = self._get_dataflow_url(
                             workspace_id=self.context.get().workspace.id,
-                            dashboard_id=dashboard_details.id,
+                            dataflow_id=dataset.id,
                         )
-                    ),
-                    project=self.get_project_name(dashboard_details=dashboard_details),
-                    displayName=dashboard_details.name,
-                    service=self.context.get().dashboard_service,
-                    owners=self.get_owner_ref(dashboard_details=dashboard_details),
-                )
-            yield Either(right=dashboard_request)
-            self.register_record(dashboard_request=dashboard_request)
-        except Exception as exc:  # pylint: disable=broad-except
+                    else:
+                        logger.warning(
+                            f"Unknown dataset type: {type(dataset)}, name: {dataset.name}"
+                        )
+                        continue
+                    data_model_request = CreateDashboardDataModelRequest(
+                        name=EntityName(dataset.id),
+                        displayName=dataset.name,
+                        description=(
+                            Markdown(dataset.description)
+                            if dataset.description
+                            else None
+                        ),
+                        service=FullyQualifiedEntityName(
+                            self.context.get().dashboard_service
+                        ),
+                        dataModelType=data_model_type,
+                        serviceType=DashboardServiceType.PowerBI.value,
+                        columns=datamodel_columns,
+                        project=self.get_project_name(dashboard_details=dataset),
+                        owners=self.get_owner_ref(dashboard_details=dataset),
+                        sourceUrl=SourceUrl(source_url),
+                    )
+                    yield Either(right=data_model_request)
+                    self.register_record_datamodel(datamodel_request=data_model_request)
+        except Exception as exc:
             yield Either(
                 left=StackTraceError(
-                    name=dashboard_details.name,
-                    error=f"Error creating dashboard [{dashboard_details}]: {exc}",
+                    name=dataset.name,
+                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
                     stackTrace=traceback.format_exc(),
                 )
             )
@@ -441,18 +668,21 @@ class PowerbiSource(DashboardServiceSource):
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
-                    name="Lineage",
+                    name="Report and Dashboard Lineage",
                     error=f"Error to yield report and dashboard lineage details: {exc}",
                     stackTrace=traceback.format_exc(),
                 )
             )
 
     def create_datamodel_report_lineage(
-        self, db_service_name: str, dashboard_details: PowerBIReport
+        self,
+        db_service_prefix: Optional[str],
+        dashboard_details: PowerBIReport,
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """
         create the lineage between datamodel and report
         """
+
         try:
             report_fqn = fqn.build(
                 self.metadata,
@@ -464,14 +694,12 @@ class PowerbiSource(DashboardServiceSource):
                 entity=Dashboard,
                 fqn=report_fqn,
             )
-
-            dataset = self._fetch_dataset_from_workspace(dashboard_details.datasetId)
-            if dataset:
+            if dashboard_details.datasetId:
                 datamodel_fqn = fqn.build(
                     self.metadata,
                     entity_type=DashboardDataModel,
                     service_name=self.context.get().dashboard_service,
-                    data_model_name=dataset.id,
+                    data_model_name=dashboard_details.datasetId,
                 )
                 datamodel_entity = self.metadata.get_by_name(
                     entity=DashboardDataModel,
@@ -482,27 +710,18 @@ class PowerbiSource(DashboardServiceSource):
                     yield self._get_add_lineage_request(
                         to_entity=report_entity, from_entity=datamodel_entity
                     )
+            else:
+                logger.debug(
+                    f"Skipping datamodel and report lineage for"
+                    f" {dashboard_details.id} as datasetId is not found"
+                )
 
-                    for table in dataset.tables or []:
-                        yield self._get_table_and_datamodel_lineage(
-                            db_service_name=db_service_name,
-                            table=table,
-                            datamodel_entity=datamodel_entity,
-                        )
-
-                    # create the lineage between table and datamodel using the pbit files
-                    if self.client.file_client:
-                        yield from self.create_table_datamodel_lineage_from_files(
-                            db_service_name=db_service_name,
-                            datamodel_entity=datamodel_entity,
-                        )
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
-                    name=f"{db_service_name} Report Lineage",
+                    name=f"Datamodel and Report Lineage",
                     error=(
-                        "Error to yield datamodel and report lineage details for DB "
-                        f"service name [{db_service_name}]: {exc}"
+                        f"Error to yield datamodel and report lineage details: {exc}"
                     ),
                     stackTrace=traceback.format_exc(),
                 )
@@ -527,41 +746,421 @@ class PowerbiSource(DashboardServiceSource):
             logger.debug(f"Error to get data_model_column_fqn {exc}")
             logger.debug(traceback.format_exc())
 
+    def parse_table_name_from_source(self, table: PowerBiTable) -> Optional[str]:
+        """
+        Parse the snowflake table name
+        """
+        try:
+            if not isinstance(table.source, list):
+                return None
+            source_expression = table.source[0].expression
+            if not source_expression:
+                logger.debug(f"No source expression found for table: {table.name}")
+                return None
+
+            if "Snowflake.Databases" in source_expression:
+                # snowflake expression
+                table_match = re.search(
+                    r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Table"\]', source_expression
+                )
+                view_match = re.search(
+                    r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="View"\]', source_expression
+                )
+                table = table_match.group(1) if table_match else None
+                view = view_match.group(1) if view_match else None
+                return table if table else view
+
+            # other general expressions
+            table_match = re.findall(r'\[Name="([^"]+)"\]', source_expression)
+            table = None
+            if isinstance(table_match, list):
+                table = table_match[1] if len(table_match) > 1 else None
+            return table
+        except Exception as exc:
+            logger.debug(f"Error to parse display table name: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_expression_regex_exp(
+        self, match: re.Match, datamodel_entity: DashboardDataModel
+    ) -> Optional[str]:
+        """parse snowflake regex expression"""
+        try:
+            if not match:
+                return None
+            elif match.group(1):
+                return match.group(1)
+            elif match.group(2):
+                dataset = self._fetch_dataset_from_workspace(datamodel_entity.name.root)
+                if dataset and dataset.expressions:
+                    # find keyword from dataset expressions
+                    for dexpression in dataset.expressions:
+                        if not dexpression.expression:
+                            logger.debug(
+                                f"No expression value found inside dataset"
+                                f"({dataset.name}) expressions' name={dexpression.name}"
+                            )
+                            continue
+                        if dexpression.name == match.group(2):
+                            pattern = r'^"([^"]+)"\s+meta'
+                            kw_match = re.search(pattern, dexpression.expression)
+                            if kw_match:
+                                return kw_match.group(1)
+        except Exception as exc:
+            logger.debug(f"Error to parse snowflake regex expression: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_redshift_source(self, source_expression: str) -> Optional[List[dict]]:
+        try:
+            db_match = re.search(
+                r'AmazonRedshift\.Database\("[^"]+","([^"]+)"\)', source_expression
+            )
+            if not db_match:
+                # not valid redshift source
+                return None
+            schema_table_match = re.findall(r'\[Name="([^"]+)"\]', source_expression)
+
+            database = db_match.group(1) if db_match else None
+            schema = table = None
+            if isinstance(schema_table_match, list):
+                schema = schema_table_match[0] if len(schema_table_match) > 0 else None
+                table = schema_table_match[1] if len(schema_table_match) > 1 else None
+
+            if table:  # atlease table should be fetched
+                return [{"database": database, "schema": schema, "table": table}]
+            return None
+        except Exception as exc:
+            logger.debug(f"Error to parse redshift table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_snowflake_query_source(
+        self, source_expression: str
+    ) -> Optional[List[dict]]:
+        """
+        Parse snowflake query source
+        source expressions like `Value.NativeQuery(Snowflake.Databases())`
+        """
+        try:
+            logger.debug(
+                f"parsing source expression through query parser: {source_expression[:100]}"
+            )
+
+            # Look for SQL query after [Data],
+            # The pattern needs to handle the concatenated strings with & operators
+            m = re.search(
+                r'\[Data\],\s*"(.+?)"(?:,\s*null|\s*\))',
+                source_expression,
+                re.IGNORECASE | re.DOTALL,
+            )
+
+            if not m:
+                logger.debug("sql query not found in source expression")
+                return None
+
+            # Extract and clean the SQL query
+            sql_query = m.group(1).replace('""', '"')
+
+            # Handle PowerBI parameter concatenation (e.g., "& Database &")
+            # For now, replace parameter references with wildcards for parsing
+            sql_query_cleaned = re.sub(r'"?\s*&\s*\w+\s*&\s*"?\.?', "%.", sql_query)
+
+            logger.debug(f"Extracted SQL query: {sql_query}")
+            logger.debug(f"Cleaned SQL query: {sql_query_cleaned}")
+
+            if not sql_query_cleaned:
+                logger.debug("Empty SQL query after extraction")
+                return None
+
+            # Clean the query for parser
+            # 1. Replace % with a placeholder database name
+            parser_query = sql_query_cleaned.replace("%", "PLACEHOLDER_DB")
+
+            # 2. Remove PowerBI line feed markers #(lf) and clean up the query
+            parser_query = parser_query.replace("#(lf)", "\n")
+
+            # 3. Remove SQL comments that might cause issues (// style comments)
+            parser_query = re.sub(r"//[^\n]*", "", parser_query)
+
+            # 4. Clean up excessive whitespace
+            parser_query = re.sub(r"\s+", " ", parser_query).strip()
+
+            logger.debug(
+                f"Attempting LineageParser with cleaned query: {parser_query[:200]}"
+            )
+
+            try:
+                parser = LineageParser(
+                    parser_query, dialect=Dialect.SNOWFLAKE, timeout_seconds=30
+                )
+            except Exception as parser_exc:
+                logger.debug(f"LineageParser failed with error: {parser_exc}")
+                logger.debug(f"Failed query was: {parser_query[:200]}...")
+                return None
+
+            if parser.source_tables:
+                logger.debug(
+                    f"LineageParser found {len(parser.source_tables)} source table(s)"
+                )
+                for table in parser.source_tables:
+                    logger.debug(
+                        f"source table: {table.raw_name}, schema: {table.schema if hasattr(table, 'schema') else 'N/A'}"
+                    )
+                lineage_tables_list = []
+                for source_table in parser.source_tables:
+                    # source_table = parser.source_tables[0]
+
+                    # Extract database from schema's parent if it exists
+                    database = None
+                    schema = None
+
+                    if hasattr(source_table, "schema") and source_table.schema:
+                        # Log what we have in the schema object
+                        logger.debug(
+                            f"Schema object type: {type(source_table.schema)}, value: {source_table.schema}"
+                        )
+
+                        # Get schema as string first
+                        schema_str = (
+                            source_table.schema.raw_name
+                            if hasattr(source_table.schema, "raw_name")
+                            else str(source_table.schema)
+                        )
+
+                        # If schema contains dots, it might be database.schema format
+                        if "." in schema_str:
+                            parts = schema_str.split(".")
+                            if len(parts) == 2:
+                                # Format: database.schema
+                                # Check for placeholder (case insensitive)
+                                database = (
+                                    parts[0]
+                                    if parts[0].upper() != "PLACEHOLDER_DB"
+                                    else None
+                                )
+                                schema = parts[1]
+                            else:
+                                # Just use as is
+                                schema = schema_str
+                        else:
+                            schema = schema_str
+                            # Check if schema has a parent (database)
+                            if (
+                                hasattr(source_table.schema, "parent")
+                                and source_table.schema.parent
+                            ):
+                                database = (
+                                    source_table.schema.parent.raw_name
+                                    if hasattr(source_table.schema.parent, "raw_name")
+                                    else str(source_table.schema.parent)
+                                )
+
+                    # Filter out placeholder values (case insensitive)
+                    if database and database.upper() == "PLACEHOLDER_DB":
+                        database = None
+
+                    table = source_table.raw_name
+
+                    if table:
+                        logger.debug(f"tables found = {database}.{schema}.{table}")
+                        lineage_tables_list.append(
+                            {
+                                "database": database,
+                                "schema": schema,
+                                "table": table,
+                            }
+                        )
+                return lineage_tables_list
+            logger.debug("tables in query not found through parser")
+            return None
+        except Exception as exc:
+            logger.debug(f"Error parsing snowflake query source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_catalog_table_definition(
+        self, source_expression: str, datamodel_entity: DashboardDataModel
+    ) -> Optional[List[dict]]:
+        """parse catalog table definition"""
+        db_match = re.search(
+            r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Database"\]', source_expression
+        )
+        schema_match = re.search(
+            r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Schema"\]', source_expression
+        )
+        table_match = re.search(
+            r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="Table"\]', source_expression
+        )
+        view_match = re.search(
+            r'\[Name=(?:"([^"]+)"|([^,]+)),Kind="View"\]', source_expression
+        )
+        try:
+            database = self._parse_expression_regex_exp(db_match, datamodel_entity)
+            schema = self._parse_expression_regex_exp(schema_match, datamodel_entity)
+            table = self._parse_expression_regex_exp(table_match, datamodel_entity)
+            view = self._parse_expression_regex_exp(view_match, datamodel_entity)
+            if table or view:  # at least table or view should be present
+                return [
+                    {
+                        "database": database,
+                        "schema": schema,
+                        "table": table if table else view,
+                    }
+                ]
+        except Exception as exc:
+            logger.debug(f"Error to parse databricks table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_databricks_source(
+        self, source_expression: str, datamodel_entity: DashboardDataModel
+    ) -> Optional[List[dict]]:
+        if "Databricks.Catalogs" not in source_expression:
+            return None
+        dataset = self._fetch_dataset_from_workspace(datamodel_entity.name.root)
+        if dataset and dataset.expressions:
+            try:
+                if DATABRICKS_QUERY_EXPRESSION_KW in source_expression:
+                    return parse_databricks_native_query_source(source_expression)
+                else:
+                    return self._parse_catalog_table_definition(
+                        source_expression, datamodel_entity
+                    )
+            except Exception as exc:
+                logger.debug(f"Error to parse databricks table source: {exc}")
+                logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_snowflake_source(
+        self, source_expression: str, datamodel_entity: DashboardDataModel
+    ) -> Optional[List[dict]]:
+        try:
+            if "Snowflake.Databases" not in source_expression:
+                # Not a snowflake valid expression
+                return None
+            if SNOWFLAKE_QUERY_EXPRESSION_KW in source_expression:
+                # snowflake query source identified
+                return self._parse_snowflake_query_source(source_expression)
+            return self._parse_catalog_table_definition(
+                source_expression, datamodel_entity
+            )
+        except Exception as exc:
+            logger.debug(f"Error to parse snowflake table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
+    def _parse_table_info_from_source_exp(
+        self, table: PowerBiTable, datamodel_entity: DashboardDataModel
+    ) -> Optional[List[dict]]:
+        try:
+            if not isinstance(table.source, list):
+                return None
+            source_expression = table.source[0].expression
+            if not source_expression:
+                logger.debug(f"No source expression found for table: {table.name}")
+                return None
+            # parse snowflake source
+            table_info_list = self._parse_snowflake_source(
+                source_expression, datamodel_entity
+            )
+            if isinstance(table_info_list, List):
+                return table_info_list
+            # parse redshift source
+            table_info_list = self._parse_redshift_source(source_expression)
+            if isinstance(table_info_list, List):
+                return table_info_list
+
+            # parse databricks source
+            table_info_list = self._parse_databricks_source(
+                source_expression, datamodel_entity
+            )
+            if isinstance(table_info_list, List):
+                return table_info_list
+
+            return None
+        except Exception as exc:
+            logger.debug(f"Error to parse table source: {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
     def _get_table_and_datamodel_lineage(
         self,
-        db_service_name: str,
+        db_service_prefix: Optional[str],
         table: PowerBiTable,
         datamodel_entity: DashboardDataModel,
-    ) -> Optional[Either[AddLineageRequest]]:
+    ) -> Iterable[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels
         """
+        (
+            prefix_service_name,
+            prefix_database_name,
+            prefix_schema_name,
+            prefix_table_name,
+        ) = self.parse_db_service_prefix(db_service_prefix)
+
         try:
-            table_fqn = fqn.build(
-                self.metadata,
-                entity_type=Table,
-                service_name=db_service_name,
-                database_name=None,
-                schema_name=None,
-                table_name=table.name,
-            )
-            table_entity = self.metadata.get_by_name(
-                entity=Table,
-                fqn=table_fqn,
+            table_info_list = self._parse_table_info_from_source_exp(
+                table, datamodel_entity
             )
+            if isinstance(table_info_list, List):
+                for table_info in table_info_list:
+                    table_name = table_info.get("table") or table.name
+                    schema_name = table_info.get("schema")
+                    database_name = table_info.get("database")
+                    if (
+                        prefix_table_name
+                        and table_name
+                        and prefix_table_name.lower() != table_name.lower()
+                    ):
+                        logger.debug(
+                            f"Table {table_name} does not match prefix {prefix_table_name}"
+                        )
+                        return
 
-            if table_entity and datamodel_entity:
-                columns_list = [column.name for column in table.columns]
-                column_lineage = self._get_column_lineage(
-                    table_entity, datamodel_entity, columns_list
-                )
-                return self._get_add_lineage_request(
-                    to_entity=datamodel_entity,
-                    from_entity=table_entity,
-                    column_lineage=column_lineage,
-                )
+                    if (
+                        prefix_schema_name
+                        and schema_name
+                        and prefix_schema_name.lower() != schema_name.lower()
+                    ):
+                        logger.debug(
+                            f"Schema {table_info.get('schema')} does not match prefix {prefix_schema_name}"
+                        )
+                        return
+
+                    if (
+                        prefix_database_name
+                        and database_name
+                        and prefix_database_name.lower() != database_name.lower()
+                    ):
+                        logger.debug(
+                            f"Database {table_info.get('database')} does not match prefix {prefix_database_name}"
+                        )
+                        return
+
+                    fqn_search_string = build_es_fqn_search_string(
+                        service_name=prefix_service_name or "*",
+                        table_name=(prefix_table_name or table_name),
+                        schema_name=(prefix_schema_name or schema_name),
+                        database_name=(prefix_database_name or database_name),
+                    )
+                    table_entity = self.metadata.search_in_any_service(
+                        entity_type=Table,
+                        fqn_search_string=fqn_search_string,
+                    )
+                    if table_entity and datamodel_entity:
+                        columns_list = [column.name for column in table.columns]
+                        column_lineage = self._get_column_lineage(
+                            table_entity, datamodel_entity, columns_list
+                        )
+                        yield self._get_add_lineage_request(
+                            to_entity=datamodel_entity,
+                            from_entity=table_entity,
+                            column_lineage=column_lineage,
+                        )
         except Exception as exc:  # pylint: disable=broad-except
-            return Either(
+            yield Either(
                 left=StackTraceError(
                     name="DataModel Lineage for pbit files",
                     error=(
@@ -571,16 +1170,17 @@ class PowerbiSource(DashboardServiceSource):
                     stackTrace=traceback.format_exc(),
                 )
             )
-        return None
 
     def create_table_datamodel_lineage_from_files(
         self,
-        db_service_name: str,
+        db_service_prefix: Optional[str],
         datamodel_entity: Optional[DashboardDataModel],
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels using pbit files
         """
+        (prefix_service_name, *_) = self.parse_db_service_prefix(db_service_prefix)
+
         try:
             # check if the datamodel_file_mappings is populated or not
             # if not, then populate the datamodel_file_mappings and process the lineage
@@ -600,8 +1200,8 @@ class PowerbiSource(DashboardServiceSource):
 
             for datamodel_schema_file in datamodel_file_list:
                 for table in datamodel_schema_file.tables or []:
-                    yield self._get_table_and_datamodel_lineage(
-                        db_service_name=db_service_name,
+                    yield from self._get_table_and_datamodel_lineage(
+                        db_service_prefix=db_service_prefix,
                         table=table,
                         datamodel_entity=datamodel_entity,
                     )
@@ -611,88 +1211,329 @@ class PowerbiSource(DashboardServiceSource):
                     name="DataModel Lineage",
                     error=(
                         "Error to yield datamodel lineage details for DB "
-                        f"service name [{db_service_name}]: {exc}"
+                        f"service name [{prefix_service_name}]: {exc}"
                     ),
                     stackTrace=traceback.format_exc(),
                 )
             )
 
-    def yield_dashboard_lineage_details(
-        self,
-        dashboard_details: Union[PowerBIDashboard, PowerBIReport],
-        db_service_name: str,
+    def create_dataset_upstream_dataflow_lineage(
+        self, datamodel: Dataset, datamodel_entity: DashboardDataModel
     ) -> Iterable[Either[AddLineageRequest]]:
         """
-        We will build the logic to build the logic as below
-        tables - datamodel - report - dashboard
+        Create lineage between dataset and upstreamDataflow
         """
-        try:
-            if isinstance(dashboard_details, PowerBIReport):
-                yield from self.create_datamodel_report_lineage(
-                    db_service_name=db_service_name, dashboard_details=dashboard_details
+        for upstream_dataflow in datamodel.upstreamDataflows or []:
+            try:
+                if not upstream_dataflow.targetDataflowId:
+                    logger.debug(
+                        f"No targetDataflowId found for upstreamDataflow in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataflow"
+                    )
+                    continue
+                upstream_dataflow_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataflow.targetDataflowId,
                 )
-
-            if isinstance(dashboard_details, PowerBIDashboard):
-                yield from self.create_report_dashboard_lineage(
-                    dashboard_details=dashboard_details
+                upstream_dataflow_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataflow_fqn,
                 )
-
-        except Exception as exc:  # pylint: disable=broad-except
-            yield Either(
-                left=StackTraceError(
-                    name="Dashboard Lineage",
-                    error=f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}",
-                    stackTrace=traceback.format_exc(),
+                if upstream_dataflow_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataflow_entity,
+                        to_entity=datamodel_entity,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataflow entity with id={str(upstream_dataflow.targetDataflowId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataset and UpstreamDataflow Lineage",
+                        error=(
+                            "Error to yield dataset and upstreamDataflow lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataflow.targetDataflowId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
                 )
+
+    def _get_downstream_data_model_column_fqn(
+        self, data_model_entity: DashboardDataModel, table_name: str, column: str
+    ) -> Optional[str]:
+        """
+        Get the FQN of the column if it exists in the
+        downstream data model entity's table and column.
+        """
+        try:
+            if not data_model_entity:
+                return None
+            for table in data_model_entity.columns:
+                if table.name.root != table_name:
+                    continue
+                for child_column in table.children or []:
+                    if column.lower() == child_column.name.root.lower():
+                        return child_column.fullyQualifiedName.root
+        except Exception as exc:
+            logger.error(
+                f"Error to get downstream data_model_column_fqn for data_model_entity="
+                f"{data_model_entity.name.root}, table_name={table_name}, column={column}: {exc}"
             )
+            logger.debug(traceback.format_exc())
+        return None
 
-    def yield_dashboard_chart(
-        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
-    ) -> Iterable[Either[CreateChartRequest]]:
-        """Get chart method
-        Args:
-            dashboard_details:
-        Returns:
-            Iterable[Chart]
+    def _create_dataset_upstream_dataset_column_lineage(
+        self,
+        datamodel_entity: DashboardDataModel,
+        upstream_dataset_entity: DashboardDataModel,
+    ) -> Optional[List[ColumnLineage]]:
         """
-        if isinstance(dashboard_details, PowerBIDashboard):
-            charts = dashboard_details.tiles
-            for chart in charts or []:
-                try:
-                    chart_title = chart.title
-                    chart_display_name = chart_title if chart_title else chart.id
-                    if filter_by_chart(
-                        self.source_config.chartFilterPattern, chart_display_name
-                    ):
-                        self.status.filter(
-                            chart_display_name, "Chart Pattern not Allowed"
+        Create column lineage between powerbi dataset/datamodel and
+        its upstream dataset/datamodel
+        """
+        try:
+            target_tables = [table.name.root for table in datamodel_entity.columns]
+            if not target_tables:
+                return []
+            column_lineage = []
+            for table in upstream_dataset_entity.columns:
+                if table.name.root not in target_tables:
+                    continue
+                for column in table.children or []:
+                    source_column = column.fullyQualifiedName.root
+                    target_column = self._get_downstream_data_model_column_fqn(
+                        data_model_entity=datamodel_entity,
+                        table_name=table.name.root,
+                        column=column.name.root,
+                    )
+                    if source_column and target_column:
+                        column_lineage.append(
+                            ColumnLineage(
+                                fromColumns=[source_column], toColumn=target_column
+                            )
                         )
-                        continue
-                    yield Either(
-                        right=CreateChartRequest(
-                            name=EntityName(chart.id),
-                            displayName=chart_display_name,
-                            chartType=ChartType.Other.value,
-                            sourceUrl=SourceUrl(
-                                self._get_chart_url(
-                                    report_id=chart.reportId,
-                                    workspace_id=self.context.get().workspace.id,
-                                    dashboard_id=dashboard_details.id,
-                                )
-                            ),
-                            service=FullyQualifiedEntityName(
-                                self.context.get().dashboard_service
-                            ),
+            return column_lineage
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.error(
+                "Error while creating column lineage between dataset = "
+                f"{datamodel_entity.name.root} and upstream dataset = "
+                f"{upstream_dataset_entity.name.root}: {exc}"
+            )
+        return []
+
+    def create_dataset_upstream_dataset_lineage(
+        self, datamodel: Dataset, datamodel_entity: DashboardDataModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Create lineage between dataset and upstreamDataset
+        """
+        for upstream_dataset in datamodel.upstreamDatasets or []:
+            try:
+                if not upstream_dataset.targetDatasetId:
+                    logger.debug(
+                        f"No targetDatasetId found for upstreamDataset in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataset"
+                    )
+                    continue
+                upstream_dataset_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataset.targetDatasetId,
+                )
+                upstream_dataset_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataset_fqn,
+                )
+                if upstream_dataset_entity and datamodel_entity:
+                    # create column lineage between current dataset/datamodel
+                    # and its upstream dataset.
+                    column_lineage = (
+                        self._create_dataset_upstream_dataset_column_lineage(
+                            datamodel_entity, upstream_dataset_entity
                         )
                     )
-                except Exception as exc:
-                    yield Either(
-                        left=StackTraceError(
-                            name=chart.title,
-                            error=f"Error creating chart [{chart.title}]: {exc}",
-                            stackTrace=traceback.format_exc(),
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataset_entity,
+                        to_entity=datamodel_entity,
+                        column_lineage=column_lineage,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataset entity with id={str(upstream_dataset.targetDatasetId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataset and UpstreamDataset Lineage",
+                        error=(
+                            "Error to yield dataset and upstreamDataset lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataset.targetDatasetId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def create_dataflow_upstream_dataflow_lineage(
+        self, datamodel: Dataflow, datamodel_entity: DashboardDataModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Create lineage between dataflow and upstreamDataflow
+        """
+        for upstream_dataflow in datamodel.upstreamDataflows or []:
+            try:
+                if not upstream_dataflow.targetDataflowId:
+                    logger.debug(
+                        f"No targetDataflowId found for upstreamDataflow in "
+                        f"datamodel [{datamodel_entity.name.root}], "
+                        f"Moving to next upstreamDataflow"
+                    )
+                    continue
+                upstream_dataflow_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=upstream_dataflow.targetDataflowId,
+                )
+                upstream_dataflow_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=upstream_dataflow_fqn,
+                )
+                if upstream_dataflow_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        from_entity=upstream_dataflow_entity,
+                        to_entity=datamodel_entity,
+                    )
+                else:
+                    logger.debug(
+                        f"No upstreamDataflow entity with id={str(upstream_dataflow.targetDataflowId)} "
+                        f"found for datamodel [{datamodel_entity.name.root}]"
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dataflow and UpstreamDataflow Lineage",
+                        error=(
+                            f"Error to yield dataflow and upstreamDataflow lineage "
+                            f"between [{datamodel_entity.name.root}, {str(upstream_dataflow.targetDataflowId)}]: {exc}"
+                        ),
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def yield_dashboard_lineage_details(
+        self,
+        dashboard_details: Group,
+        db_service_prefix: Optional[str] = None,
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        We will build the logic to build the logic as below
+        tables - datamodel - report - dashboard
+        """
+        (prefix_service_name, *_) = self.parse_db_service_prefix(db_service_prefix)
+
+        for dashboard in self.filtered_dashboards or []:
+            dashboard_details = self.get_dashboard_details(dashboard)
+            try:
+                if isinstance(dashboard_details, PowerBIReport):
+                    yield from self.create_datamodel_report_lineage(
+                        db_service_prefix=db_service_prefix,
+                        dashboard_details=dashboard_details,
+                    )
+                if isinstance(dashboard_details, PowerBIDashboard):
+                    yield from self.create_report_dashboard_lineage(
+                        dashboard_details=dashboard_details
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Dashboard Lineage",
+                        error=f"Error to yield dashboard lineage details for DB service name [{str(prefix_service_name)}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+        """
+        Iterate loop for filtered datamodels so datamodels which are not connected to 
+        any report but have tables would be eligible for a dataset-db_table lineage.
+        Also create below lineages: 
+        1. dataset-db_table
+        2. dataset-upstreamDataflow
+        3. dataset-upstreamDataset
+        4. dataflow-upstreamDataflow
+        """
+        for datamodel in self.filtered_datamodels or []:
+            try:
+                datamodel_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.context.get().dashboard_service,
+                    data_model_name=datamodel.id,
+                )
+                datamodel_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=datamodel_fqn,
+                )
+                if datamodel_entity:
+                    if isinstance(datamodel, Dataset):
+                        # 1. datamodel-db_table lineage
+                        for table in datamodel.tables or []:
+                            yield from self._get_table_and_datamodel_lineage(
+                                db_service_prefix=db_service_prefix,
+                                table=table,
+                                datamodel_entity=datamodel_entity,
+                            )
+                        # 2. dataset-upstreamDataflow lineage
+                        yield from self.create_dataset_upstream_dataflow_lineage(
+                            datamodel, datamodel_entity
+                        )
+                        # 3. dataset-upstreamDataset lineage
+                        yield from self.create_dataset_upstream_dataset_lineage(
+                            datamodel, datamodel_entity
                         )
+                        # create the lineage between table and datamodel using the pbit files
+                        if self.client.file_client:
+                            yield from self.create_table_datamodel_lineage_from_files(
+                                db_service_prefix=db_service_prefix,
+                                datamodel_entity=datamodel_entity,
+                            )
+                    elif isinstance(datamodel, Dataflow):
+                        # create dataflow-upstreamDataflow lineage
+                        yield from self.create_dataflow_upstream_dataflow_lineage(
+                            datamodel, datamodel_entity
+                        )
+                    else:
+                        logger.warning(
+                            f"Unknown datamodel type: {type(datamodel)}, name: {datamodel.name}"
+                        )
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name="Datamodel Lineage",
+                        error=f"Error to yield datamodel lineage details for DB service name [{str(prefix_service_name)}]: {exc}",
+                        stackTrace=traceback.format_exc(),
                     )
+                )
+
+    def yield_datamodel_dashboard_lineage(
+        self,
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Returns:
+            Lineage request between Data Models and Dashboards
+        """
+        """
+            We're implementing this differently inside `yield_dashboard_lineage_details`
+            since we have report and dashboard both as dashboard.
+        """
 
     def _fetch_dataset_from_workspace(
         self, dataset_id: Optional[str]
@@ -734,21 +1575,6 @@ class PowerbiSource(DashboardServiceSource):
                     return report_data
         return None
 
-    def _fetch_dataset_workspace(self, dataset_id: Optional[str]) -> Optional[str]:
-        """
-        Method to search the workspace name in which the dataset in contained
-        """
-        if dataset_id:
-            workspace_names = (
-                workspace.name
-                for workspace in self.workspace_data
-                for dataset in workspace.datasets
-                if dataset.id == dataset_id
-            )
-            return next(iter(workspace_names), None)
-
-        return None
-
     def get_project_name(self, dashboard_details: Any) -> Optional[str]:
         """
         Get the project / workspace / folder / collection name of the dashboard
@@ -761,3 +1587,98 @@ class PowerbiSource(DashboardServiceSource):
                 f"Error fetching project name for {dashboard_details.id}: {exc}"
             )
         return None
+
+    def get_owner_ref(  # pylint: disable=unused-argument, useless-return
+        self, dashboard_details: Any
+    ) -> Optional[EntityReferenceList]:
+        """
+        Method to process the dashboard owners
+        """
+        try:
+            if not self.source_config.includeOwners:
+                logger.debug(
+                    f"Skipping owner processing for {dashboard_details.id} "
+                    f"as includeOwners is False"
+                )
+                return None
+            owner_ref_list = []  # to assign multiple owners to entity if they exist
+            for owner in dashboard_details.users or []:
+                owner_ref = None
+                # put filtering conditions
+                if isinstance(dashboard_details, Dataset):
+                    access_right = owner.datasetUserAccessRight
+                elif isinstance(dashboard_details, Dataflow):
+                    access_right = owner.dataflowUserAccessRight
+                elif isinstance(dashboard_details, PowerBIReport):
+                    access_right = owner.reportUserAccessRight
+                elif isinstance(dashboard_details, PowerBIDashboard):
+                    access_right = owner.dashboardUserAccessRight
+
+                if owner.userType != "Member":
+                    logger.debug(
+                        f"User is not a member of {dashboard_details.id}:"
+                        f" ({owner.displayName}, {owner.email})"
+                    )
+                    continue
+                if access_right and any(
+                    keyword in access_right.lower()
+                    for keyword in OWNER_ACCESS_RIGHTS_KEYWORDS
+                ):
+                    if owner.email:
+                        try:
+                            owner_email = EmailStr._validate(owner.email)
+                        except PydanticCustomError:
+                            logger.debug(f"Invalid email for owner: {owner.email}")
+                            owner_email = None
+                        if owner_email:
+                            try:
+                                owner_ref = self.metadata.get_reference_by_email(
+                                    owner_email.lower()
+                                )
+                            except Exception as err:
+                                logger.debug(
+                                    f"Could not process owner data with email"
+                                    f" {owner.email} in {dashboard_details.id}: {err}"
+                                )
+                    elif owner.displayName:
+                        try:
+                            owner_ref = self.metadata.get_reference_by_name(
+                                name=owner.displayName
+                            )
+                        except Exception as err:
+                            logger.debug(
+                                f"Could not process owner data with name"
+                                f" {owner.displayName} in {dashboard_details.id}: {err}"
+                            )
+                    if owner_ref:
+                        owner_ref_list.append(owner_ref.root[0])
+                else:
+                    logger.debug(
+                        f"User does not have owner, admin or write access to"
+                        f" {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                    )
+            # check for last modified, configuredBy user
+            current_active_user = None
+            if isinstance(dashboard_details, Dataset):
+                current_active_user = dashboard_details.configuredBy
+            elif isinstance(dashboard_details, (Dataflow, PowerBIReport)):
+                current_active_user = dashboard_details.modifiedBy
+            if current_active_user:
+                try:
+                    owner_ref = self.metadata.get_reference_by_email(
+                        current_active_user.lower()
+                    )
+                    if owner_ref and owner_ref.root[0] not in owner_ref_list:
+                        owner_ref_list.append(owner_ref.root[0])
+                except Exception as err:
+                    logger.debug(f"Could not fetch current active user due to {err}")
+            if len(owner_ref_list) > 0:
+                logger.debug(
+                    f"Successfully fetched owners data for {dashboard_details.id}"
+                )
+                return EntityReferenceList(root=owner_ref_list)
+            return None
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
+        return None