openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/ingestion/sink/metadata_rest.py

@@ -21,7 +21,6 @@ from pydantic import BaseModel
 from requests.exceptions import HTTPError
 
 from metadata.config.common import ConfigModel
-from metadata.data_insight.source.metadata import DataInsightRecord
 from metadata.data_quality.api.models import TestCaseResultResponse, TestCaseResults
 from metadata.generated.schema.analytics.reportData import ReportData
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -51,6 +50,7 @@ from metadata.generated.schema.tests.testCaseResolutionStatus import (
     TestCaseResolutionStatus,
 )
 from metadata.generated.schema.tests.testSuite import TestSuite
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.schema import Topic
 from metadata.ingestion.api.models import Either, Entity, StackTraceError
 from metadata.ingestion.api.steps import Sink
@@ -59,6 +59,7 @@ from metadata.ingestion.models.data_insight import OMetaDataInsightSample
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
 from metadata.ingestion.models.patch_request import (
     ALLOWED_COMMON_PATCH_FIELDS,
@@ -123,7 +124,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         metadata: OpenMetadata,
         pipeline_name: Optional[str] = None,
     ):
-        config = MetadataRestSinkConfig.parse_obj(config_dict)
+        config = MetadataRestSinkConfig.model_validate(config_dict)
         return cls(config, metadata)
 
     @property
@@ -186,6 +187,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
             restrict_update_fields=RESTRICT_UPDATE_LIST,
             array_entity_fields=ARRAY_ENTITY_FIELDS,
+            override_metadata=record.override_metadata,
         )
         patched_entity = PatchedEntity(new_entity=entity) if entity else None
         return Either(right=patched_entity)
@@ -249,13 +251,51 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
+    @_run_dispatch.register
+    def write_override_lineage(
+        self, add_lineage: OMetaLineageRequest
+    ) -> Either[Dict[str, Any]]:
+        """
+        Writes the override lineage for the given lineage request.
+
+        Args:
+            add_lineage (OMetaLineageRequest): The lineage request containing the override lineage information.
+
+        Returns:
+            Either[Dict[str, Any]]: The result of the dispatch operation.
+        """
+        if (
+            add_lineage.override_lineage is True
+            and add_lineage.lineage_request.edge.lineageDetails
+            and add_lineage.lineage_request.edge.lineageDetails.source
+        ):
+            if (
+                add_lineage.lineage_request.edge.lineageDetails.pipeline
+                and add_lineage.lineage_request.edge.lineageDetails.source
+                == LineageSource.PipelineLineage
+            ):
+                self.metadata.delete_lineage_by_source(
+                    entity_type="pipeline",
+                    entity_id=str(
+                        add_lineage.lineage_request.edge.lineageDetails.pipeline.id.root
+                    ),
+                    source=add_lineage.lineage_request.edge.lineageDetails.source.value,
+                )
+            else:
+                self.metadata.delete_lineage_by_source(
+                    entity_type=add_lineage.lineage_request.edge.toEntity.type,
+                    entity_id=str(add_lineage.lineage_request.edge.toEntity.id.root),
+                    source=add_lineage.lineage_request.edge.lineageDetails.source.value,
+                )
+        return self._run_dispatch(add_lineage.lineage_request)
+
     def _create_role(self, create_role: CreateRoleRequest) -> Optional[Role]:
         """
         Internal helper method for write_user
         """
         try:
             role = self.metadata.create_or_update(create_role)
-            self.role_entities[role.name] = str(role.id.__root__)
+            self.role_entities[role.name] = str(role.id.root)
             return role
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -269,7 +309,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         """
         try:
             team = self.metadata.create_or_update(create_team)
-            self.team_entities[team.name.__root__] = str(team.id.__root__)
+            self.team_entities[team.name.root] = str(team.id.root)
             return team
         except Exception as exc:
             logger.debug(traceback.format_exc())
@@ -292,7 +332,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             for role in record.roles:
                 try:
                     role_entity = self.metadata.get_by_name(
-                        entity=Role, fqn=str(role.name.__root__)
+                        entity=Role, fqn=str(role.name.root)
                     )
                 except APIError:
                     role_entity = self._create_role(role)
@@ -307,18 +347,16 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             for team in record.teams:
                 try:
                     team_entity = self.metadata.get_by_name(
-                        entity=Team, fqn=str(team.name.__root__)
+                        entity=Team, fqn=str(team.name.root)
                     )
                     if not team_entity:
                         raise APIError(
-                            error={
-                                "message": f"Creating a new team {team.name.__root__}"
-                            }
+                            error={"message": f"Creating a new team {team.name.root}"}
                         )
-                    team_ids.append(team_entity.id.__root__)
+                    team_ids.append(team_entity.id.root)
                 except APIError:
                     team_entity = self._create_team(team)
-                    team_ids.append(team_entity.id.__root__)
+                    team_ids.append(team_entity.id.root)
                 except Exception as exc:
                     logger.debug(traceback.format_exc())
                     logger.warning(f"Unexpected error writing team [{team}]: {exc}")
@@ -326,7 +364,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             team_ids = None
 
         # Update user data with the new Role and Team IDs
-        user_profile = record.user.dict(exclude_unset=True)
+        user_profile = record.user.model_dump(exclude_unset=True)
         user_profile["roles"] = role_ids
         user_profile["teams"] = team_ids
         metadata_user = CreateUserRequest(**user_profile)
@@ -421,10 +459,10 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         """Write the test case result"""
         res = self.metadata.add_test_case_results(
             test_results=record.testCaseResult,
-            test_case_fqn=record.testCase.fullyQualifiedName.__root__,
+            test_case_fqn=record.testCase.fullyQualifiedName.root,
         )
         logger.debug(
-            f"Successfully ingested test case results for test case {record.testCase.name.__root__}"
+            f"Successfully ingested test case results for test case {record.testCase.name.root}"
         )
         return Either(right=res)
 
@@ -449,20 +487,12 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         )
         return Either(right=record.record)
 
-    @_run_dispatch.register
-    def write_data_insight(self, record: DataInsightRecord) -> Either[ReportData]:
-        """
-        Use the /dataQuality/testCases endpoint to ingest sample test suite
-        """
-        self.metadata.add_data_insight_report_data(record.data)
-        return Either(left=None, right=record.data)
-
     @_run_dispatch.register
     def write_data_insight_kpi(self, record: KpiResult) -> Either[KpiResult]:
         """
         Use the /dataQuality/testCases endpoint to ingest sample test suite
         """
-        self.metadata.add_kpi_result(fqn=record.kpiFqn.__root__, record=record)
+        self.metadata.add_kpi_result(fqn=record.kpiFqn.root, record=record)
         return Either(left=None, right=record)
 
     @_run_dispatch.register
@@ -532,7 +562,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             profile_request=record.profile,
         )
         logger.debug(
-            f"Successfully ingested profile metrics for {record.table.fullyQualifiedName.__root__}"
+            f"Successfully ingested profile metrics for {record.table.fullyQualifiedName.root}"
         )
 
         if record.sample_data:
@@ -542,13 +572,13 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             if not table_data:
                 self.status.failed(
                     StackTraceError(
-                        name=table.fullyQualifiedName.__root__,
+                        name=table.fullyQualifiedName.root,
                         error="Error trying to ingest sample data for table",
                     )
                 )
             else:
                 logger.debug(
-                    f"Successfully ingested sample data for {record.table.fullyQualifiedName.__root__}"
+                    f"Successfully ingested sample data for {record.table.fullyQualifiedName.root}"
                 )
 
         if record.column_tags:
@@ -558,13 +588,13 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
             if not patched:
                 self.status.failed(
                     StackTraceError(
-                        name=table.fullyQualifiedName.__root__,
+                        name=table.fullyQualifiedName.root,
                         error="Error patching tags for table",
                     )
                 )
             else:
                 logger.debug(
-                    f"Successfully patched tag {record.column_tags} for {record.table.fullyQualifiedName.__root__}"
+                    f"Successfully patched tag {record.column_tags} for {record.table.fullyQualifiedName.root}"
                 )
 
         return Either(right=table)
@@ -586,7 +616,7 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
         for result in record.test_results or []:
             self.metadata.add_test_case_results(
                 test_results=result.testCaseResult,
-                test_case_fqn=result.testCase.fullyQualifiedName.__root__,
+                test_case_fqn=result.testCase.fullyQualifiedName.root,
             )
             self.status.scanned(result)
 

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -15,7 +15,8 @@ import traceback
 from abc import ABC, abstractmethod
 from typing import Any, Iterable, List, Optional, Set, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
+from typing_extensions import Annotated
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
@@ -40,6 +41,7 @@ from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipelin
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import Uuid
 from metadata.generated.schema.type.entityLineage import (
     ColumnLineage,
     EntitiesEdge,
@@ -47,6 +49,7 @@ from metadata.generated.schema.type.entityLineage import (
 )
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.api.delete import delete_entity_from_source
 from metadata.ingestion.api.models import Either, Entity
@@ -55,6 +58,7 @@ from metadata.ingestion.api.topology_runner import C, TopologyRunnerMixin
 from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.patch_request import PatchRequest
 from metadata.ingestion.models.topology import (
     NodeStage,
@@ -96,7 +100,9 @@ class DashboardServiceTopology(ServiceTopology):
     data that has been produced by any parent node.
     """
 
-    root = TopologyNode(
+    root: Annotated[
+        TopologyNode, Field(description="Root node for the topology")
+    ] = TopologyNode(
         producer="get_services",
         stages=[
             NodeStage(
@@ -122,7 +128,9 @@ class DashboardServiceTopology(ServiceTopology):
     # handles them as independent entities.
     # When configuring a new source, we will either implement
     # the yield_bulk_datamodel or yield_datamodel functions.
-    bulk_data_model = TopologyNode(
+    bulk_data_model: Annotated[
+        TopologyNode, Field(description="Write data models in bulk")
+    ] = TopologyNode(
         producer="list_datamodels",
         stages=[
             NodeStage(
@@ -134,7 +142,9 @@ class DashboardServiceTopology(ServiceTopology):
             )
         ],
     )
-    dashboard = TopologyNode(
+    dashboard: Annotated[
+        TopologyNode, Field(description="Process dashboards")
+    ] = TopologyNode(
         producer="get_dashboard",
         stages=[
             NodeStage(
@@ -196,7 +206,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     config: WorkflowSource
     metadata: OpenMetadata
     # Big union of types we want to fetch dynamically
-    service_connection: DashboardConnection.__fields__["config"].type_
+    service_connection: DashboardConnection.model_fields["config"].annotation
 
     topology = DashboardServiceTopology()
     context = TopologyContextManager(topology)
@@ -211,7 +221,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         super().__init__()
         self.config = config
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
         self.source_config: DashboardServiceMetadataPipeline = (
             self.config.sourceConfig.config
         )
@@ -321,7 +331,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 except Exception as err:
                     logger.debug(traceback.format_exc())
                     logger.error(
-                        f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
+                        f"Error to yield dashboard lineage details for data model name [{str(datamodel)}]: {err}"
                     )
 
     def get_db_service_names(self) -> List[str]:
@@ -336,17 +346,25 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     def yield_dashboard_lineage(
         self, dashboard_details: Any
-    ) -> Iterable[Either[AddLineageRequest]]:
+    ) -> Iterable[Either[OMetaLineageRequest]]:
         """
         Yields lineage if config is enabled.
 
         We will look for the data in all the services
         we have informed.
         """
-        yield from self.yield_datamodel_dashboard_lineage() or []
+        for lineage in self.yield_datamodel_dashboard_lineage() or []:
+            if lineage.right is not None:
+                yield Either(
+                    right=OMetaLineageRequest(
+                        lineage_request=lineage.right,
+                        override_lineage=self.source_config.overrideLineage,
+                    )
+                )
+            else:
+                yield lineage
 
         db_service_names = self.get_db_service_names()
-
         for db_service_name in db_service_names or []:
             yield from self.yield_dashboard_lineage_details(
                 dashboard_details, db_service_name
@@ -416,7 +434,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     def get_owner_ref(  # pylint: disable=unused-argument, useless-return
         self, dashboard_details
-    ) -> Optional[EntityReference]:
+    ) -> Optional[EntityReferenceList]:
         """
         Method to process the dashboard owners
         """
@@ -432,8 +450,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         dashboard_fqn = fqn.build(
             self.metadata,
             entity_type=Dashboard,
-            service_name=dashboard_request.service.__root__,
-            dashboard_name=dashboard_request.name.__root__,
+            service_name=dashboard_request.service.root,
+            dashboard_name=dashboard_request.name.root,
         )
 
         self.dashboard_source_state.add(dashboard_fqn)
@@ -447,8 +465,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         datamodel_fqn = fqn.build(
             self.metadata,
             entity_type=DashboardDataModel,
-            service_name=datamodel_request.service.__root__,
-            data_model_name=datamodel_request.name.__root__,
+            service_name=datamodel_request.service.root,
+            data_model_name=datamodel_request.name.root,
         )
 
         self.datamodel_source_state.add(datamodel_fqn)
@@ -464,11 +482,11 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 right=AddLineageRequest(
                     edge=EntitiesEdge(
                         fromEntity=EntityReference(
-                            id=from_entity.id.__root__,
+                            id=Uuid(from_entity.id.root),
                             type=LINEAGE_MAP[type(from_entity)],
                         ),
                         toEntity=EntityReference(
-                            id=to_entity.id.__root__,
+                            id=Uuid(to_entity.id.root),
                             type=LINEAGE_MAP[type(to_entity)],
                         ),
                         lineageDetails=LineageDetails(
@@ -492,7 +510,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
             return None
         for tbl_column in data_model_entity.columns:
             if tbl_column.displayName.lower() == column.lower():
-                return tbl_column.fullyQualifiedName.__root__
+                return tbl_column.fullyQualifiedName.root
         return None
 
     def get_dashboard(self) -> Any:
@@ -541,28 +559,6 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     def prepare(self):
         """By default, nothing to prepare"""
 
-    def fqn_from_context(self, stage: NodeStage, entity_name: C) -> str:
-        """
-        We are overriding this method since CreateDashboardDataModelRequest needs to add an extra value to the context
-        names.
-
-        Read the context
-        :param stage: Topology node being processed
-        :param entity_request: Request sent to the sink
-        :return: Entity FQN derived from context
-        """
-        context_names = [
-            self.context.get().__dict__[dependency]
-            for dependency in stage.consumer or []  # root nodes do not have consumers
-        ]
-
-        if isinstance(stage.type_, DashboardDataModel):
-            context_names.append("model")
-
-        return fqn._build(  # pylint: disable=protected-access
-            *context_names, entity_name
-        )
-
     def check_database_schema_name(self, database_schema_name: str):
         """
         Check if the input database schema name is equal to "<default>" and return the input name if it is not.
@@ -599,7 +595,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
         """
         patch_request = PatchRequest(
             original_entity=original_entity,
-            new_entity=original_entity.copy(update=create_request.__dict__),
+            new_entity=original_entity.model_copy(update=create_request.__dict__),
         )
         if isinstance(original_entity, Dashboard):
             # For patch the charts need to be entity ref instead of fqn
@@ -609,11 +605,13 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 if chart_entity:
                     charts_entity_ref_list.append(
                         EntityReference(
-                            id=chart_entity.id.__root__,
+                            id=chart_entity.id.root,
                             type=LINEAGE_MAP[type(chart_entity)],
                         )
                     )
-            patch_request.new_entity.charts = charts_entity_ref_list
+            patch_request.new_entity.charts = EntityReferenceList(
+                charts_entity_ref_list
+            )
 
             # For patch the datamodels need to be entity ref instead of fqn
             datamodel_entity_ref_list = []
@@ -624,11 +622,13 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 if datamodel_entity:
                     datamodel_entity_ref_list.append(
                         EntityReference(
-                            id=datamodel_entity.id.__root__,
+                            id=datamodel_entity.id.root,
                             type=LINEAGE_MAP[type(datamodel_entity)],
                         )
                     )
-            patch_request.new_entity.dataModels = datamodel_entity_ref_list
+            patch_request.new_entity.dataModels = EntityReferenceList(
+                datamodel_entity_ref_list
+            )
         return patch_request
 
     def _get_column_lineage(

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -38,7 +38,13 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -64,8 +70,8 @@ class DomodashboardSource(DashboardServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: DomoDashboardConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: DomoDashboardConnection = config.serviceConnection.root.config
         if not isinstance(connection, DomoDashboardConnection):
             raise InvalidSourceException(
                 f"Expected DomoDashboardConnection, but got {connection}"
@@ -97,7 +103,7 @@ class DomodashboardSource(DashboardServiceSource):
 
     def get_owner_ref(
         self, dashboard_details: DomoDashboardDetails
-    ) -> Optional[EntityReference]:
+    ) -> Optional[EntityReferenceList]:
         for owner in dashboard_details.owners or []:
             try:
                 owner_details = self.client.domo.users_get(owner.id)
@@ -114,25 +120,29 @@ class DomodashboardSource(DashboardServiceSource):
     ) -> Iterable[Either[CreateDashboardRequest]]:
         try:
             dashboard_url = (
-                f"{self.service_connection.instanceDomain}/page/{dashboard_details.id}"
+                f"{self.service_connection.instanceDomain}page/{dashboard_details.id}"
             )
 
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.id,
-                sourceUrl=dashboard_url,
+                name=EntityName(dashboard_details.id),
+                sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=dashboard_details.description,
+                description=Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
                 service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -217,10 +227,12 @@ class DomodashboardSource(DashboardServiceSource):
                 if chart.name:
                     yield Either(
                         right=CreateChartRequest(
-                            name=chart_id,
-                            description=chart.description,
+                            name=EntityName(chart_id),
+                            description=Markdown(chart.description)
+                            if chart.description
+                            else None,
                             displayName=chart.name,
-                            sourceUrl=chart_url,
+                            sourceUrl=SourceUrl(chart_url),
                             service=self.context.get().dashboard_service,
                             chartType=get_standard_chart_type(chart.metadata.chartType),
                         )

metadata/ingestion/source/dashboard/lightdash/client.py

@@ -34,7 +34,7 @@ class LightdashApiClient:
     def __init__(self, config):
         self.config = config
         client_config = ClientConfig(
-            base_url=self.config.hostPort,
+            base_url=str(self.config.hostPort),
             api_version="",
             access_token=self.config.apiKey.get_secret_value(),
             auth_header="Authorization",

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -26,6 +26,12 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -53,8 +59,8 @@ class LightdashSource(DashboardServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: LightdashConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: LightdashConnection = config.serviceConnection.root.config
         if not isinstance(connection, LightdashConnection):
             raise InvalidSourceException(
                 f"Expected LightdashConnection, but got {connection}"
@@ -105,21 +111,25 @@ class LightdashSource(DashboardServiceSource):
                 f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
             )
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.uuid,
-                sourceUrl=dashboard_url,
+                name=EntityName(dashboard_details.uuid),
+                sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=dashboard_details.description,
+                description=Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
                 service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield dashboard_request
             self.register_record(dashboard_request=dashboard_request)
@@ -150,10 +160,12 @@ class LightdashSource(DashboardServiceSource):
                     self.status.filter(chart.name, "Chart Pattern not allowed")
                     continue
                 yield CreateChartRequest(
-                    name=chart.uuid,
+                    name=EntityName(chart.uuid),
                     displayName=chart.name,
-                    description=chart.description,
-                    sourceUrl=chart_url,
+                    description=Markdown(chart.description)
+                    if chart.description
+                    else None,
+                    sourceUrl=SourceUrl(chart_url),
                     service=self.context.get().dashboard_service,
                 )
                 self.status.scanned(chart.name)