openmetadata-ingestion 1.4.7.2__py3-none-any.whl → 1.5.0.0__py3-none-any.whl

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -15,7 +15,8 @@ import traceback
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Optional, Set, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+from typing_extensions import Annotated
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -40,6 +41,7 @@ from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipelin
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import Uuid
 from metadata.generated.schema.type.entityLineage import (
     ColumnLineage,
     EntitiesEdge,
@@ -47,6 +49,7 @@ from metadata.generated.schema.type.entityLineage import (
 )
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.api.delete import delete_entity_from_source
 from metadata.ingestion.api.models import Either, Entity
@@ -55,6 +58,7 @@ from metadata.ingestion.api.topology_runner import C, TopologyRunnerMixin
 from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.patch_request import PatchRequest
 from metadata.ingestion.models.topology import (
     NodeStage,
@@ -96,7 +100,9 @@ class DashboardServiceTopology(ServiceTopology):
     data that has been produced by any parent node.
     """
 
-    root = TopologyNode(
+    root: Annotated[
+        TopologyNode, Field(description="Root node for the topology")
+    ] = TopologyNode(
         producer="get_services",
         stages=[
             NodeStage(
@@ -122,7 +128,9 @@ class DashboardServiceTopology(ServiceTopology):
     # handles them as independent entities.
     # When configuring a new source, we will either implement
     # the yield_bulk_datamodel or yield_datamodel functions.
-    bulk_data_model = TopologyNode(
+    bulk_data_model: Annotated[
+        TopologyNode, Field(description="Write data models in bulk")
+    ] = TopologyNode(
         producer="list_datamodels",
         stages=[
             NodeStage(
@@ -134,7 +142,9 @@ class DashboardServiceTopology(ServiceTopology):
             )
         ],
     )
-    dashboard = TopologyNode(
+    dashboard: Annotated[
+        TopologyNode, Field(description="Process dashboards")
+    ] = TopologyNode(
         producer="get_dashboard",
         stages=[
             NodeStage(
@@ -196,7 +206,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     config: WorkflowSource
     metadata: OpenMetadata
     # Big union of types we want to fetch dynamically
-    service_connection: DashboardConnection.__fields__["config"].type_
+    service_connection: DashboardConnection.model_fields["config"].annotation
 
     topology = DashboardServiceTopology()
     context = TopologyContextManager(topology)
@@ -211,7 +221,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         super().__init__()
         self.config = config
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
         self.source_config: DashboardServiceMetadataPipeline = (
             self.config.sourceConfig.config
         )
@@ -321,7 +331,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 except Exception as err:
                     logger.debug(traceback.format_exc())
                     logger.error(
-                        f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
+                        f"Error to yield dashboard lineage details for data model name [{str(datamodel)}]: {err}"
                     )
 
     def get_db_service_names(self) -> List[str]:
@@ -336,17 +346,25 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     def yield_dashboard_lineage(
         self, dashboard_details: Any
-    ) -> Iterable[Either[AddLineageRequest]]:
+    ) -> Iterable[Either[OMetaLineageRequest]]:
         """
         Yields lineage if config is enabled.
 
         We will look for the data in all the services
         we have informed.
         """
-        yield from self.yield_datamodel_dashboard_lineage() or []
+        for lineage in self.yield_datamodel_dashboard_lineage() or []:
+            if lineage.right is not None:
+                yield Either(
+                    right=OMetaLineageRequest(
+                        lineage_request=lineage.right,
+                        override_lineage=self.source_config.overrideLineage,
+                    )
+                )
+            else:
+                yield lineage
 
         db_service_names = self.get_db_service_names()
-
         for db_service_name in db_service_names or []:
             yield from self.yield_dashboard_lineage_details(
                 dashboard_details, db_service_name
@@ -416,7 +434,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     def get_owner_ref(  # pylint: disable=unused-argument, useless-return
         self, dashboard_details
-    ) -> Optional[EntityReference]:
+    ) -> Optional[EntityReferenceList]:
         """
         Method to process the dashboard owners
         """
@@ -432,8 +450,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         dashboard_fqn = fqn.build(
             self.metadata,
             entity_type=Dashboard,
-            service_name=dashboard_request.service.__root__,
-            dashboard_name=dashboard_request.name.__root__,
+            service_name=dashboard_request.service.root,
+            dashboard_name=dashboard_request.name.root,
         )
 
         self.dashboard_source_state.add(dashboard_fqn)
@@ -447,8 +465,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         datamodel_fqn = fqn.build(
             self.metadata,
             entity_type=DashboardDataModel,
-            service_name=datamodel_request.service.__root__,
-            data_model_name=datamodel_request.name.__root__,
+            service_name=datamodel_request.service.root,
+            data_model_name=datamodel_request.name.root,
         )
 
         self.datamodel_source_state.add(datamodel_fqn)
@@ -464,11 +482,11 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 right=AddLineageRequest(
                     edge=EntitiesEdge(
                         fromEntity=EntityReference(
-                            id=from_entity.id.__root__,
+                            id=Uuid(from_entity.id.root),
                             type=LINEAGE_MAP[type(from_entity)],
                         ),
                         toEntity=EntityReference(
-                            id=to_entity.id.__root__,
+                            id=Uuid(to_entity.id.root),
                             type=LINEAGE_MAP[type(to_entity)],
                         ),
                         lineageDetails=LineageDetails(
@@ -492,7 +510,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
             return None
         for tbl_column in data_model_entity.columns:
             if tbl_column.displayName.lower() == column.lower():
-                return tbl_column.fullyQualifiedName.__root__
+                return tbl_column.fullyQualifiedName.root
         return None
 
     def get_dashboard(self) -> Any:
@@ -541,28 +559,6 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     def prepare(self):
         """By default, nothing to prepare"""
 
-    def fqn_from_context(self, stage: NodeStage, entity_name: C) -> str:
-        """
-        We are overriding this method since CreateDashboardDataModelRequest needs to add an extra value to the context
-        names.
-
-        Read the context
-        :param stage: Topology node being processed
-        :param entity_request: Request sent to the sink
-        :return: Entity FQN derived from context
-        """
-        context_names = [
-            self.context.get().__dict__[dependency]
-            for dependency in stage.consumer or []  # root nodes do not have consumers
-        ]
-
-        if isinstance(stage.type_, DashboardDataModel):
-            context_names.append("model")
-
-        return fqn._build(  # pylint: disable=protected-access
-            *context_names, entity_name
-        )
-
     def check_database_schema_name(self, database_schema_name: str):
         """
         Check if the input database schema name is equal to "<default>" and return the input name if it is not.
@@ -599,7 +595,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         patch_request = PatchRequest(
             original_entity=original_entity,
-            new_entity=original_entity.copy(update=create_request.__dict__),
+            new_entity=original_entity.model_copy(update=create_request.__dict__),
         )
         if isinstance(original_entity, Dashboard):
             # For patch the charts need to be entity ref instead of fqn
@@ -609,11 +605,13 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 if chart_entity:
                     charts_entity_ref_list.append(
                         EntityReference(
-                            id=chart_entity.id.__root__,
+                            id=chart_entity.id.root,
                             type=LINEAGE_MAP[type(chart_entity)],
                         )
                     )
-            patch_request.new_entity.charts = charts_entity_ref_list
+            patch_request.new_entity.charts = EntityReferenceList(
+                charts_entity_ref_list
+            )
 
             # For patch the datamodels need to be entity ref instead of fqn
             datamodel_entity_ref_list = []
@@ -624,11 +622,13 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 if datamodel_entity:
                     datamodel_entity_ref_list.append(
                         EntityReference(
-                            id=datamodel_entity.id.__root__,
+                            id=datamodel_entity.id.root,
                             type=LINEAGE_MAP[type(datamodel_entity)],
                         )
                     )
-            patch_request.new_entity.dataModels = datamodel_entity_ref_list
+            patch_request.new_entity.dataModels = EntityReferenceList(
+                datamodel_entity_ref_list
+            )
         return patch_request
 
     def _get_column_lineage(

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -38,7 +38,13 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -64,8 +70,8 @@ class DomodashboardSource(DashboardServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: DomoDashboardConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: DomoDashboardConnection = config.serviceConnection.root.config
         if not isinstance(connection, DomoDashboardConnection):
             raise InvalidSourceException(
                 f"Expected DomoDashboardConnection, but got {connection}"
@@ -97,7 +103,7 @@ class DomodashboardSource(DashboardServiceSource):
 
     def get_owner_ref(
         self, dashboard_details: DomoDashboardDetails
-    ) -> Optional[EntityReference]:
+    ) -> Optional[EntityReferenceList]:
         for owner in dashboard_details.owners or []:
             try:
                 owner_details = self.client.domo.users_get(owner.id)
@@ -114,25 +120,29 @@ class DomodashboardSource(DashboardServiceSource):
     ) -> Iterable[Either[CreateDashboardRequest]]:
         try:
             dashboard_url = (
-                f"{self.service_connection.instanceDomain}/page/{dashboard_details.id}"
+                f"{self.service_connection.instanceDomain}page/{dashboard_details.id}"
             )
 
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.id,
-                sourceUrl=dashboard_url,
+                name=EntityName(dashboard_details.id),
+                sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=dashboard_details.description,
+                description=Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
                 service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -175,7 +185,7 @@ class DomodashboardSource(DashboardServiceSource):
             pages = self.client.domo.page_get(page_id)
             return DomoDashboardDetails(
                 name=pages["name"],
-                id=pages["id"],
+                id=str(pages["id"]),
                 cardIds=pages.get("cardIds", []),
                 description=pages.get("description", ""),
                 collectionIds=pages.get("collectionIds", []),
@@ -207,7 +217,7 @@ class DomodashboardSource(DashboardServiceSource):
             try:
                 chart = self.client.custom.get_chart_details(page_id=chart_id)
                 chart_url = (
-                    f"{self.service_connection.instanceDomain}/page/"
+                    f"{self.service_connection.instanceDomain}page/"
                     f"{dashboard_details.id}/kpis/details/{chart_id}"
                 )
 
@@ -217,10 +227,12 @@ class DomodashboardSource(DashboardServiceSource):
                 if chart.name:
                     yield Either(
                         right=CreateChartRequest(
-                            name=chart_id,
-                            description=chart.description,
+                            name=EntityName(str(chart_id)),
+                            description=Markdown(chart.description)
+                            if chart.description
+                            else None,
                             displayName=chart.name,
-                            sourceUrl=chart_url,
+                            sourceUrl=SourceUrl(chart_url),
                             service=self.context.get().dashboard_service,
                             chartType=get_standard_chart_type(chart.metadata.chartType),
                         )

metadata/ingestion/source/dashboard/lightdash/client.py

@@ -34,7 +34,7 @@ class LightdashApiClient:
     def __init__(self, config):
         self.config = config
         client_config = ClientConfig(
-            base_url=self.config.hostPort,
+            base_url=str(self.config.hostPort),
             api_version="",
             access_token=self.config.apiKey.get_secret_value(),
             auth_header="Authorization",

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -26,6 +26,12 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -53,8 +59,8 @@ class LightdashSource(DashboardServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: LightdashConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: LightdashConnection = config.serviceConnection.root.config
         if not isinstance(connection, LightdashConnection):
             raise InvalidSourceException(
                 f"Expected LightdashConnection, but got {connection}"
@@ -105,21 +111,25 @@ class LightdashSource(DashboardServiceSource):
                 f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
             )
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.uuid,
-                sourceUrl=dashboard_url,
+                name=EntityName(dashboard_details.uuid),
+                sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=dashboard_details.description,
+                description=Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
                 service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield dashboard_request
             self.register_record(dashboard_request=dashboard_request)
@@ -150,10 +160,12 @@ class LightdashSource(DashboardServiceSource):
                     self.status.filter(chart.name, "Chart Pattern not allowed")
                     continue
                 yield CreateChartRequest(
-                    name=chart.uuid,
+                    name=EntityName(chart.uuid),
                     displayName=chart.name,
-                    description=chart.description,
-                    sourceUrl=chart_url,
+                    description=Markdown(chart.description)
+                    if chart.description
+                    else None,
+                    sourceUrl=SourceUrl(chart_url),
                     service=self.context.get().dashboard_service,
                 )
                 self.status.scanned(chart.name)

metadata/ingestion/source/dashboard/lightdash/models.py

@@ -13,34 +13,34 @@ class LightdashChart(BaseModel):
     name: str
     organizationUuid: str
     uuid: str
-    description: Optional[str]
+    description: Optional[str] = None
     projectUuid: str
     spaceUuid: str
-    pinnedListUuid: Optional[str]
+    pinnedListUuid: Optional[str] = None
     spaceName: str
-    chartType: Optional[str]
-    dashboardUuid: Optional[str]
-    dashboardName: Optional[str]
+    chartType: Optional[str] = None
+    dashboardUuid: Optional[str] = None
+    dashboardName: Optional[str] = None
 
 
 class LightdashDashboard(BaseModel):
     organizationUuid: str
     name: str
-    description: Optional[str]
+    description: Optional[str] = None
     uuid: str
     projectUuid: str
     updatedAt: str
     spaceUuid: str
     views: float
     firstViewedAt: str
-    pinnedListUuid: Optional[str]
-    pinnedListOrder: Optional[float]
-    charts: Optional[List[LightdashChart]]
+    pinnedListUuid: Optional[str] = None
+    pinnedListOrder: Optional[float] = None
+    charts: Optional[List[LightdashChart]] = None
 
 
 class LightdashChartList(BaseModel):
-    charts: Optional[List[LightdashChart]]
+    charts: Optional[List[LightdashChart]] = None
 
 
 class LightdashDashboardList(BaseModel):
-    dashboards: Optional[List[LightdashDashboard]]
+    dashboards: Optional[List[LightdashDashboard]] = None

metadata/ingestion/source/dashboard/looker/bulk_parser.py

@@ -67,7 +67,7 @@ class BulkLkmlParser(metaclass=Singleton):
         file_paths = self.reader.get_local_files(search_key=".view.lkml")
         for _path in file_paths:
             file = self._read_file(Includes(_path))
-            lkml_file = LkmlFile.parse_obj(lkml.load(file))
+            lkml_file = LkmlFile.model_validate(lkml.load(file))
             self.parsed_files[Includes(_path)] = file
             for view in lkml_file.views:
                 view.source_file = _path
@@ -120,7 +120,7 @@ class BulkLkmlParser(metaclass=Singleton):
         """
         if isinstance(self.reader, ApiReader):
             return (
-                f"Parser at [{self.reader.credentials.repositoryOwner.__root__}/"
-                f"{self.reader.credentials.repositoryName.__root__}]"
+                f"Parser at [{self.reader.credentials.repositoryOwner.root}/"
+                f"{self.reader.credentials.repositoryName.root}]"
             )
         return f"Parser at [{self.reader}]"

metadata/ingestion/source/dashboard/looker/connection.py

@@ -39,7 +39,7 @@ def get_connection(connection: LookerConnection) -> Looker40SDK:
             "LOOKERSDK_CLIENT_SECRET"
         ] = connection.clientSecret.get_secret_value()
     if not os.environ.get("LOOKERSDK_BASE_URL"):
-        os.environ["LOOKERSDK_BASE_URL"] = connection.hostPort
+        os.environ["LOOKERSDK_BASE_URL"] = str(connection.hostPort)
 
     return looker_sdk.init40()