openmetadata-ingestion 1.7.1.5__py3-none-any.whl → 1.7.2.1__py3-none-any.whl

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-06-09T12:47:28+00:00
+#   timestamp: 2025-06-12T10:55:12+00:00
 
 from __future__ import annotations
 

metadata/ingestion/api/topology_runner.py

@@ -393,32 +393,31 @@ class TopologyRunnerMixin(Generic[C]):
         if entity is None and stage.use_cache:
             # check if we find the entity in the entities list
             entity_source_hash = self.cache[stage.type_].get(entity_fqn)
-            if entity_source_hash:
-                # if the source hash is present, compare it with new hash
-                # if overrideMetadata is true, we will always update the entity
-                if (
-                    entity_source_hash != create_entity_request_hash
-                    or self.source_config.overrideMetadata
-                ):
-                    # the entity has changed, get the entity from server and make a patch request
-                    entity = self.metadata.get_by_name(
-                        entity=stage.type_,
-                        fqn=entity_fqn,
-                        fields=["*"],
-                    )
+            # if the source hash is not present or different from new hash, update the entity
+            # if overrideMetadata is true, we will always update the entity
+            if (
+                entity_source_hash != create_entity_request_hash
+                or self.source_config.overrideMetadata
+            ):
+                # the entity has changed, get the entity from server and make a patch request
+                entity = self.metadata.get_by_name(
+                    entity=stage.type_,
+                    fqn=entity_fqn,
+                    fields=["*"],
+                )
 
-                    # we return the entity for a patch update
-                    if entity:
-                        patch_entity = self.create_patch_request(
-                            original_entity=entity, create_request=entity_request.right
-                        )
-                        entity_request.right = patch_entity
-                else:
-                    # nothing has changed on the source skip the API call
-                    logger.debug(
-                        f"No changes detected for {str(stage.type_.__name__)} '{entity_fqn}'"
+                # we return the entity for a patch update
+                if entity:
+                    patch_entity = self.create_patch_request(
+                        original_entity=entity, create_request=entity_request.right
                     )
-                    same_fingerprint = True
+                    entity_request.right = patch_entity
+            else:
+                # nothing has changed on the source skip the API call
+                logger.debug(
+                    f"No changes detected for {str(stage.type_.__name__)} '{entity_fqn}'"
+                )
+                same_fingerprint = True
 
         if not same_fingerprint:
             # We store the generated source hash and yield the request

metadata/ingestion/models/custom_pydantic.py

@@ -57,6 +57,8 @@ class BaseModel(PydanticBaseModel):
             if not self.__class__.__name__.endswith("Connection"):
                 # Only parse FilterPattern for Connection classes
                 return
+            if not hasattr(self, "__pydantic_fields__"):
+                return
             for field in self.__pydantic_fields__:
                 if field.endswith("FilterPattern"):
                     from metadata.generated.schema.type.filterPattern import (

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -113,6 +113,7 @@ class OMetaLineageMixin(Generic[T]):
         Add lineage relationship between two entities and returns
         the entity information of the origin node
         """
+        data = deepcopy(data)
         try:
             patch_op_success = False
             if check_patch and data.edge.lineageDetails:

metadata/ingestion/source/dashboard/lightdash/client.py

@@ -18,6 +18,7 @@ from metadata.ingestion.ometa.client import REST, ClientConfig
 from metadata.ingestion.source.dashboard.lightdash.models import (
     LightdashChart,
     LightdashDashboard,
+    LightdashSpace,
 )
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import utils_logger
@@ -28,6 +29,7 @@ logger = utils_logger()
 class LightdashApiClient:
     """
     REST Auth & Client for Lightdash
+    API Documentation: https://docs.lightdash.com/api-reference/v1
     """
 
     client: REST
@@ -50,6 +52,44 @@ class LightdashApiClient:
             "/api/v1/org",
         )
 
+    def get_spaces(self) -> List[LightdashSpace]:
+        """GET Lightdash Spaces within the project"""
+        try:
+            response = self.client.get(
+                f"api/v1/projects/{self.config.projectUUID}/spaces"
+            )
+            response_json_results = response.get("results")
+            if response_json_results is None:
+                logger.warning(
+                    "Failed to fetch the spaces list for the Lightdash Connector"
+                )
+                return []
+
+            if len(response_json_results) > 0:
+                spaces_list = []
+                for space in response_json_results:
+                    spaces_list.append(LightdashSpace(**space))
+                return spaces_list
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                "Failed to fetch the spaces list for the Lightdash Connector"
+            )
+        return []
+
+    def get_project_name(self, project_uuid: str) -> str:
+        """GET project name"""
+        try:
+            response = self.client.get(f"api/v1/projects/{project_uuid}")
+            response_json_results = response.get("results")
+            return response_json_results["name"]
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                "Failed to fetch the project data from the Lightdash Connector"
+            )
+            return ""
+
     def get_charts_list(self) -> List[LightdashChart]:
         """
         Get List of all charts
@@ -93,12 +133,15 @@ class LightdashApiClient:
                 )
                 return []
 
+            space_name = results["name"]
             dashboards_raw = results["dashboards"]
 
             if len(dashboards_raw) > 0:
                 dashboards_list = []
                 for dashboard in dashboards_raw:
-                    dashboards_list.append(LightdashDashboard(**dashboard))
+                    dashboards_list.append(
+                        LightdashDashboard(**dashboard, spaceName=space_name)
+                    )
 
                 self.add_dashboard_lineage(dashboards_list=dashboards_list)
                 return dashboards_list
@@ -109,8 +152,10 @@ class LightdashApiClient:
             )
         return []
 
-    def add_dashboard_lineage(self, dashboards_list) -> None:
-        charts_uuid_list = []
+    def add_dashboard_lineage(self, dashboards_list: List[LightdashDashboard]) -> None:
+        """
+        Get Lineage of all dashboard charts
+        """
         for dashboard in dashboards_list:
             response = self.client.get(f"api/v1/dashboards/{dashboard.uuid}")
             response_json_results = response.get("results")
@@ -122,14 +167,42 @@ class LightdashApiClient:
                 return
 
             charts = response_json_results["tiles"]
-            charts_properties = [chart["properties"] for chart in charts]
-
+            # Lightdash has title, loom & markdown chart types which we want to ignore
+            accepted_chart_types = ["saved_chart", "sql_chart", "semantic_viewer_chart"]
+            charts_properties = [
+                chart["properties"]
+                for chart in charts
+                if chart["type"] in accepted_chart_types
+            ]
+
+            dashboard_external_uuid_charts = []
+            dashboard_internal_charts = []
             for chart in charts_properties:
-                charts_uuid_list.append(chart["savedChartUuid"])
-
-            dashboard.charts = self.get_charts_objects(charts_uuid_list)
+                if chart["belongsToDashboard"]:
+                    dashboard_internal_charts.append(
+                        LightdashChart(
+                            uuid=chart["savedChartUuid"],
+                            name=chart["chartName"],
+                            organizationUuid=dashboard.organizationUuid,
+                            projectUuid=dashboard.projectUuid,
+                            spaceUuid=dashboard.spaceUuid,
+                            spaceName=dashboard.spaceName,
+                            chartType=chart["lastVersionChartKind"],
+                            chartKind=chart["lastVersionChartKind"],
+                        )
+                    )
+                else:
+                    dashboard_external_uuid_charts.append(chart["savedChartUuid"])
+
+            dashboard_external_charts = self.get_charts_objects(
+                dashboard_external_uuid_charts
+            )
+            dashboard.charts = dashboard_external_charts + dashboard_internal_charts
 
     def get_charts_objects(self, charts_uuid_list) -> List[LightdashChart]:
+        """
+        Get Lineage of all non-dashboard charts
+        """
         all_charts = self.get_charts_list()
         charts_objects = []
 

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -11,7 +11,7 @@
 """Lightdash source module"""
 
 import traceback
-from typing import Iterable, List, Optional
+from typing import Any, Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -32,16 +32,18 @@ from metadata.generated.schema.type.basic import (
     Markdown,
     SourceUrl,
 )
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.lightdash.models import (
     LightdashChart,
     LightdashDashboard,
+    LightdashSpace,
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
-from metadata.utils.helpers import clean_uri, replace_special_with
+from metadata.utils.helpers import clean_uri, get_standard_chart_type
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -73,12 +75,17 @@ class LightdashSource(DashboardServiceSource):
         metadata: OpenMetadata,
     ):
         super().__init__(config, metadata)
+        self.spaces: List[LightdashSpace] = []
         self.charts: List[LightdashChart] = []
 
     def prepare(self):
+        self.spaces = self.client.get_spaces()
         self.charts = self.client.get_charts_list()
         return super().prepare()
 
+    def get_project_name(self, dashboard_details: Any) -> Optional[str]:
+        return self.client.get_project_name(dashboard_details.projectUuid)
+
     def get_dashboards_list(self) -> Optional[List[LightdashDashboard]]:
         """
         Get List of all dashboards
@@ -101,18 +108,21 @@ class LightdashSource(DashboardServiceSource):
 
     def yield_dashboard(
         self, dashboard_details: LightdashDashboard
-    ) -> Iterable[CreateDashboardRequest]:
+    ) -> Iterable[Either[CreateDashboardRequest]]:
         """
         Method to Get Dashboard Entity
         """
         try:
             dashboard_url = (
-                f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.uuid}-"
-                f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
+                f"{clean_uri(self.service_connection.hostPort)}/projects/{dashboard_details.projectUuid}"
+                f"/dashboards/{dashboard_details.uuid}/view"
             )
+            dashboard_charts = [chart.uuid for chart in dashboard_details.charts]
+
             dashboard_request = CreateDashboardRequest(
                 name=EntityName(dashboard_details.uuid),
                 sourceUrl=SourceUrl(dashboard_url),
+                project=self.client.get_project_name(dashboard_details.projectUuid),
                 displayName=dashboard_details.name,
                 description=Markdown(dashboard_details.description)
                 if dashboard_details.description
@@ -126,12 +136,12 @@ class LightdashSource(DashboardServiceSource):
                             chart_name=chart,
                         )
                     )
-                    for chart in self.context.get().charts or []
+                    for chart in dashboard_charts or []
                 ],
                 service=self.context.get().dashboard_service,
                 owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
-            yield dashboard_request
+            yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())
@@ -140,8 +150,8 @@ class LightdashSource(DashboardServiceSource):
             )
 
     def yield_dashboard_chart(
-        self, dashboard_details: LightdashChart
-    ) -> Iterable[Optional[CreateChartRequest]]:
+        self, dashboard_details: LightdashDashboard
+    ) -> Iterable[Either[CreateChartRequest]]:
         """Get chart method
 
         Args:
@@ -149,24 +159,32 @@ class LightdashSource(DashboardServiceSource):
         Returns:
             Iterable[CreateChartRequest]
         """
-        charts = self.charts
-        for chart in charts:
+        # charts = self.charts
+        logger.info(
+            f"Processing ChartRequests for dashboard {dashboard_details.spaceName}:{dashboard_details.name}"
+        )
+        for chart in dashboard_details.charts:
             try:
                 chart_url = (
-                    f"{clean_uri(self.service_connection.hostPort)}/question/{chart.uuid}-"
-                    f"{replace_special_with(raw=chart.name.lower(), replacement='-')}"
+                    f"{clean_uri(self.service_connection.hostPort)}/projects/{dashboard_details.projectUuid}"
+                    f"/saved/{chart.uuid}"
                 )
+                chart_type = get_standard_chart_type(chart.chartKind).value
+
                 if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
                     self.status.filter(chart.name, "Chart Pattern not allowed")
                     continue
-                yield CreateChartRequest(
-                    name=EntityName(chart.uuid),
-                    displayName=chart.name,
-                    description=Markdown(chart.description)
-                    if chart.description
-                    else None,
-                    sourceUrl=SourceUrl(chart_url),
-                    service=self.context.get().dashboard_service,
+                yield Either(
+                    right=CreateChartRequest(
+                        name=EntityName(chart.uuid),
+                        displayName=chart.name,
+                        description=Markdown(chart.description)
+                        if chart.description
+                        else None,
+                        sourceUrl=SourceUrl(chart_url),
+                        service=self.context.get().dashboard_service,
+                        chartType=chart_type,
+                    )
                 )
                 self.status.scanned(chart.name)
             except Exception as exc:  # pylint: disable=broad-except

metadata/ingestion/source/dashboard/lightdash/models.py

@@ -19,6 +19,7 @@ class LightdashChart(BaseModel):
     pinnedListUuid: Optional[str] = None
     spaceName: str
     chartType: Optional[str] = None
+    chartKind: Optional[str] = None
     dashboardUuid: Optional[str] = None
     dashboardName: Optional[str] = None
 
@@ -31,6 +32,7 @@ class LightdashDashboard(BaseModel):
     projectUuid: str
     updatedAt: str
     spaceUuid: str
+    spaceName: Optional[str] = None
     views: float
     firstViewedAt: str
     pinnedListUuid: Optional[str] = None
@@ -38,6 +40,15 @@ class LightdashDashboard(BaseModel):
     charts: Optional[List[LightdashChart]] = None
 
 
+class LightdashSpace(BaseModel):
+    organizationUuid: str
+    projectUuid: str
+    uuid: str
+    name: str
+    isPrivate: bool
+    parentSpaceUuid: Optional[str] = None
+
+
 class LightdashChartList(BaseModel):
     charts: Optional[List[LightdashChart]] = None
 

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -80,6 +80,8 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+OWNER_ACCESS_RIGHTS_KEYWORDS = ["owner", "write", "admin"]
+
 
 class PowerbiSource(DashboardServiceSource):
     """PowerBi Source Class"""
@@ -1017,42 +1019,49 @@ class PowerbiSource(DashboardServiceSource):
                 elif isinstance(dashboard_details, PowerBIDashboard):
                     access_right = owner.dashboardUserAccessRight
 
-                if owner.userType != "Member" or (
-                    isinstance(
-                        dashboard_details, (Dataflow, PowerBIReport, PowerBIDashboard)
-                    )
-                    and access_right != "Owner"
-                ):
+                if owner.userType != "Member":
                     logger.warning(
-                        f"User is not a member and has no access to the {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                        f"User is not a member of {dashboard_details.id}:"
+                        f" ({owner.displayName}, {owner.email})"
                     )
                     continue
-                if owner.email:
-                    try:
-                        owner_email = EmailStr._validate(owner.email)
-                    except PydanticCustomError:
-                        logger.warning(f"Invalid email for owner: {owner.email}")
-                        owner_email = None
-                    if owner_email:
+                if access_right and any(
+                    keyword in access_right.lower()
+                    for keyword in OWNER_ACCESS_RIGHTS_KEYWORDS
+                ):
+                    if owner.email:
+                        try:
+                            owner_email = EmailStr._validate(owner.email)
+                        except PydanticCustomError:
+                            logger.warning(f"Invalid email for owner: {owner.email}")
+                            owner_email = None
+                        if owner_email:
+                            try:
+                                owner_ref = self.metadata.get_reference_by_email(
+                                    owner_email.lower()
+                                )
+                            except Exception as err:
+                                logger.warning(
+                                    f"Could not process owner data with email"
+                                    f" {owner.email} in {dashboard_details.id}: {err}"
+                                )
+                    elif owner.displayName:
                         try:
-                            owner_ref = self.metadata.get_reference_by_email(
-                                owner_email.lower()
+                            owner_ref = self.metadata.get_reference_by_name(
+                                name=owner.displayName
                             )
                         except Exception as err:
                             logger.warning(
-                                f"Could not fetch owner data with email {owner.email} in {dashboard_details.id}: {err}"
+                                f"Could not process owner data with name"
+                                f" {owner.displayName} in {dashboard_details.id}: {err}"
                             )
-                elif owner.displayName:
-                    try:
-                        owner_ref = self.metadata.get_reference_by_name(
-                            name=owner.displayName
-                        )
-                    except Exception as err:
-                        logger.warning(
-                            f"Could not process owner data with name {owner.displayName} in {dashboard_details.id}: {err}"
-                        )
-                if owner_ref:
-                    owner_ref_list.append(owner_ref.root[0])
+                    if owner_ref:
+                        owner_ref_list.append(owner_ref.root[0])
+                else:
+                    logger.warning(
+                        f"User does not have owner, admin or write access to"
+                        f" {dashboard_details.id}: ({owner.displayName}, {owner.email})"
+                    )
             # check for last modified, configuredBy user
             current_active_user = None
             if isinstance(dashboard_details, Dataset):

metadata/ingestion/source/dashboard/tableau/client.py

@@ -13,7 +13,7 @@ Wrapper module of TableauServerConnection client
 """
 import math
 import traceback
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import validators
 from cached_property import cached_property
@@ -77,16 +77,18 @@ class TableauClient:
         self,
         tableau_server_auth: Union[PersonalAccessTokenAuth, TableauAuth],
         config,
-        verify_ssl,
+        verify_ssl: Union[bool, str],
         pagination_limit: int,
     ):
         self.tableau_server = Server(str(config.hostPort), use_server_version=True)
+        if config.apiVersion:
+            self.tableau_server.version = config.apiVersion
         self.tableau_server.add_http_options({"verify": verify_ssl})
         self.tableau_server.auth.sign_in(tableau_server_auth)
         self.config = config
         self.pagination_limit = pagination_limit
         self.custom_sql_table_queries: Dict[str, List[str]] = {}
-        self.usage_metrics: Dict[str, int] = {}
+        self.owner_cache: Dict[str, TableauOwner] = {}
 
     @cached_property
     def server_info(self) -> Callable:
@@ -101,9 +103,15 @@ class TableauClient:
 
     def get_tableau_owner(self, owner_id: str) -> Optional[TableauOwner]:
         try:
+            if owner_id in self.owner_cache:
+                return self.owner_cache[owner_id]
             owner = self.tableau_server.users.get_by_id(owner_id) if owner_id else None
             if owner and owner.email:
-                return TableauOwner(id=owner.id, name=owner.name, email=owner.email)
+                owner_obj = TableauOwner(
+                    id=owner.id, name=owner.name, email=owner.email
+                )
+                self.owner_cache[owner_id] = owner_obj
+                return owner_obj
         except Exception as err:
             logger.debug(f"Failed to fetch owner details for ID {owner_id}: {str(err)}")
         return None
@@ -130,21 +138,20 @@ class TableauClient:
                 )
                 view_count += view.total_views
             except AttributeError as e:
-                logger.warning(
+                logger.debug(
                     f"Failed to process view due to missing attribute: {str(e)}"
                 )
                 continue
             except Exception as e:
-                logger.warning(f"Failed to process view: {str(e)}")
+                logger.debug(f"Failed to process view: {str(e)}")
                 continue
 
         return charts, view_count
 
-    def get_workbooks(self) -> List[TableauDashboard]:
+    def get_workbooks(self) -> Iterable[TableauDashboard]:
         """
         Fetch all tableau workbooks
         """
-        workbooks: Optional[List[TableauDashboard]] = []
         self.cache_custom_sql_tables()
         for workbook in Pager(self.tableau_server.workbooks):
             try:
@@ -152,22 +159,20 @@ class TableauClient:
                 charts, user_views = self.get_workbook_charts_and_user_count(
                     workbook.views
                 )
-                workbooks.append(
-                    TableauDashboard(
-                        id=workbook.id,
-                        name=workbook.name,
-                        project=TableauBaseModel(
-                            id=workbook.project_id, name=workbook.project_name
-                        ),
-                        description=workbook.description,
-                        owner=self.get_tableau_owner(workbook.owner_id),
-                        tags=workbook.tags,
-                        webpageUrl=workbook.webpage_url,
-                        charts=charts,
-                        dataModels=self.get_datasources(dashboard_id=workbook.id),
-                        user_views=user_views,
-                    )
+                workbook = TableauDashboard(
+                    id=workbook.id,
+                    name=workbook.name,
+                    project=TableauBaseModel(
+                        id=workbook.project_id, name=workbook.project_name
+                    ),
+                    owner=self.get_tableau_owner(workbook.owner_id),
+                    description=workbook.description,
+                    tags=workbook.tags,
+                    webpageUrl=workbook.webpage_url,
+                    charts=charts,
+                    user_views=user_views,
                 )
+                yield workbook
             except AttributeError as err:
                 logger.warning(
                     f"Failed to process workbook due to missing attribute: {str(err)}"
@@ -176,7 +181,6 @@ class TableauClient:
             except Exception as err:
                 logger.warning(f"Failed to process workbook: {str(err)}")
                 continue
-        return workbooks
 
     def test_get_workbooks(self):
         for workbook in Pager(self.tableau_server.workbooks):