openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/dashboard/mode/metadata.py

@@ -29,6 +29,12 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.lineage.parser import LineageParser
@@ -55,15 +61,15 @@ class ModeSource(DashboardServiceSource):
         metadata: OpenMetadata,
     ):
         super().__init__(config, metadata)
-        self.workspace_name = config.serviceConnection.__root__.config.workspaceName
+        self.workspace_name = config.serviceConnection.root.config.workspaceName
         self.data_sources = self.client.get_all_data_sources(self.workspace_name)
 
     @classmethod
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: ModeConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: ModeConnection = config.serviceConnection.root.config
         if not isinstance(connection, ModeConnection):
             raise InvalidSourceException(
                 f"Expected ModeConnection, but got {connection}"
@@ -97,21 +103,25 @@ class ModeSource(DashboardServiceSource):
         dashboard_path = dashboard_details[client.LINKS][client.SHARE][client.HREF]
         dashboard_url = f"{clean_uri(self.service_connection.hostPort)}{dashboard_path}"
         dashboard_request = CreateDashboardRequest(
-            name=dashboard_details.get(client.TOKEN),
-            sourceUrl=dashboard_url,
+            name=EntityName(dashboard_details.get(client.TOKEN)),
+            sourceUrl=SourceUrl(dashboard_url),
             displayName=dashboard_details.get(client.NAME),
-            description=dashboard_details.get(client.DESCRIPTION),
+            description=Markdown(dashboard_details.get(client.DESCRIPTION))
+            if dashboard_details.get(client.DESCRIPTION)
+            else None,
             charts=[
-                fqn.build(
-                    self.metadata,
-                    entity_type=Chart,
-                    service_name=self.context.get().dashboard_service,
-                    chart_name=chart,
+                FullyQualifiedEntityName(
+                    fqn.build(
+                        self.metadata,
+                        entity_type=Chart,
+                        service_name=self.context.get().dashboard_service,
+                        chart_name=chart,
+                    )
                 )
                 for chart in self.context.get().charts or []
             ],
             service=self.context.get().dashboard_service,
-            owner=self.get_owner_ref(dashboard_details=dashboard_details),
+            owners=self.get_owner_ref(dashboard_details=dashboard_details),
         )
         yield Either(right=dashboard_request)
         self.register_record(dashboard_request=dashboard_request)
@@ -201,10 +211,10 @@ class ModeSource(DashboardServiceSource):
                     )
                     yield Either(
                         right=CreateChartRequest(
-                            name=chart.get(client.TOKEN),
+                            name=EntityName(chart.get(client.TOKEN)),
                             displayName=chart_name,
                             chartType=ChartType.Other,
-                            sourceUrl=chart_url,
+                            sourceUrl=SourceUrl(chart_url),
                             service=self.context.get().dashboard_service,
                         )
                     )

metadata/ingestion/source/dashboard/mstr/client.py

@@ -167,7 +167,7 @@ class MSTRClient:
             dashboards = []
             for result in results:
                 dashboards.append(
-                    MstrDashboard(projectName=project_name, **result.dict())
+                    MstrDashboard(projectName=project_name, **result.model_dump())
                 )
 
             dashboards_list = MstrDashboardList(dashboards=dashboards)

metadata/ingestion/source/dashboard/mstr/metadata.py

@@ -25,8 +25,13 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    SourceUrl,
+)
 from metadata.ingestion.api.models import Either
-from metadata.ingestion.api.source import InvalidSourceException
+from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
 from metadata.ingestion.source.dashboard.mstr.models import (
@@ -54,8 +59,8 @@ class MstrSource(DashboardServiceSource):
         metadata: OpenMetadata,
         pipeline_name: Optional[str] = None,
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: MstrConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: MstrConnection = config.serviceConnection.root.config
         if not isinstance(connection, MstrConnection):
             raise InvalidSourceException(
                 f"Expected MstrConnection, but got {connection}"
@@ -107,21 +112,23 @@ class MstrSource(DashboardServiceSource):
                 f"{dashboard_details.projectId}/{dashboard_details.id}"
             )
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.id,
+                name=EntityName(dashboard_details.id),
                 displayName=dashboard_details.name,
-                sourceUrl=dashboard_url,
+                sourceUrl=SourceUrl(dashboard_url),
                 project=dashboard_details.projectName,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
                 service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)

metadata/ingestion/source/dashboard/mstr/models.py

@@ -43,7 +43,7 @@ class MstrProject(BaseModel):
 
 
 class MstrProjectList(BaseModel):
-    projects: Optional[List[MstrProject]]
+    projects: Optional[List[MstrProject]] = None
 
 
 class MstrSearchResult(BaseModel):
@@ -54,7 +54,7 @@ class MstrSearchResult(BaseModel):
     name: str
     id: str
     type: int
-    description: Optional[str]
+    description: Optional[str] = None
     subtype: int
     dateCreated: str
     dateModified: str
@@ -80,7 +80,7 @@ class MstrDashboard(BaseModel):
     name: str
     id: str
     type: int
-    description: Optional[str]
+    description: Optional[str] = None
     subtype: int
     dateCreated: str
     dateModified: str
@@ -96,7 +96,7 @@ class MstrDashboard(BaseModel):
 
 
 class MstrDashboardList(BaseModel):
-    dashboards: Optional[List[MstrDashboard]]
+    dashboards: Optional[List[MstrDashboard]] = None
 
 
 class MstrAttribute(BaseModel):

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -18,7 +18,7 @@ from time import sleep
 from typing import List, Optional, Tuple
 
 import msal
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
     PowerBIConnection,
@@ -327,8 +327,7 @@ class PowerBiApiClient:
 
 
 class PowerBiClient(BaseModel):
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     api_client: PowerBiApiClient
     file_client: Optional[PowerBiFileClient]

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -41,6 +41,12 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -96,7 +102,8 @@ class PowerbiSource(DashboardServiceSource):
 
     def close(self):
         self.metadata.close()
-        self.client.file_client.delete_tmp_files()
+        if self.client.file_client:
+            self.client.file_client.delete_tmp_files()
 
     def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
         """
@@ -205,8 +212,8 @@ class PowerbiSource(DashboardServiceSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: PowerBIConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: PowerBIConnection = config.serviceConnection.root.config
         if not isinstance(connection, PowerBIConnection):
             raise InvalidSourceException(
                 f"Expected PowerBIConnection, but got {connection}"
@@ -328,10 +335,12 @@ class PowerbiSource(DashboardServiceSource):
         """
         try:
             data_model_request = CreateDashboardDataModelRequest(
-                name=dataset.id,
+                name=EntityName(dataset.id),
                 displayName=dataset.name,
-                description=dataset.description,
-                service=self.context.get().dashboard_service,
+                description=Markdown(dataset.description)
+                if dataset.description
+                else None,
+                service=FullyQualifiedEntityName(self.context.get().dashboard_service),
                 dataModelType=DataModelType.PowerBIDataModel.value,
                 serviceType=DashboardServiceType.PowerBI.value,
                 columns=self._get_column_info(dataset),
@@ -404,38 +413,46 @@ class PowerbiSource(DashboardServiceSource):
         try:
             if isinstance(dashboard_details, PowerBIDashboard):
                 dashboard_request = CreateDashboardRequest(
-                    name=dashboard_details.id,
-                    sourceUrl=self._get_dashboard_url(
-                        workspace_id=self.context.get().workspace.id,
-                        dashboard_id=dashboard_details.id,
+                    name=EntityName(dashboard_details.id),
+                    sourceUrl=SourceUrl(
+                        self._get_dashboard_url(
+                            workspace_id=self.context.get().workspace.id,
+                            dashboard_id=dashboard_details.id,
+                        )
                     ),
                     project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.displayName,
                     dashboardType=DashboardType.Dashboard,
                     charts=[
-                        fqn.build(
-                            self.metadata,
-                            entity_type=Chart,
-                            service_name=self.context.get().dashboard_service,
-                            chart_name=chart,
+                        FullyQualifiedEntityName(
+                            fqn.build(
+                                self.metadata,
+                                entity_type=Chart,
+                                service_name=self.context.get().dashboard_service,
+                                chart_name=chart,
+                            )
                         )
                         for chart in self.context.get().charts or []
                     ],
-                    service=self.context.get().dashboard_service,
-                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                    service=FullyQualifiedEntityName(
+                        self.context.get().dashboard_service
+                    ),
+                    owners=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             else:
                 dashboard_request = CreateDashboardRequest(
-                    name=dashboard_details.id,
+                    name=EntityName(dashboard_details.id),
                     dashboardType=DashboardType.Report,
-                    sourceUrl=self._get_report_url(
-                        workspace_id=self.context.get().workspace.id,
-                        dashboard_id=dashboard_details.id,
+                    sourceUrl=SourceUrl(
+                        self._get_report_url(
+                            workspace_id=self.context.get().workspace.id,
+                            dashboard_id=dashboard_details.id,
+                        )
                     ),
                     project=self.get_project_name(dashboard_details=dashboard_details),
                     displayName=dashboard_details.name,
                     service=self.context.get().dashboard_service,
-                    owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                    owners=self.get_owner_ref(dashboard_details=dashboard_details),
                 )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -564,8 +581,8 @@ class PowerbiSource(DashboardServiceSource):
                 return None
             for tbl_column in data_model_entity.columns:
                 for child_column in tbl_column.children or []:
-                    if column.lower() == child_column.name.__root__.lower():
-                        return child_column.fullyQualifiedName.__root__
+                    if column.lower() == child_column.name.root.lower():
+                        return child_column.fullyQualifiedName.root
             return None
         except Exception as exc:
             logger.debug(f"Error to get data_model_column_fqn {exc}")
@@ -576,7 +593,7 @@ class PowerbiSource(DashboardServiceSource):
         db_service_name: str,
         table: PowerBiTable,
         datamodel_entity: DashboardDataModel,
-    ) -> Optional[Iterable[Either[AddLineageRequest]]]:
+    ) -> Optional[Either[AddLineageRequest]]:
         """
         Method to create lineage between table and datamodels
         """
@@ -728,15 +745,19 @@ class PowerbiSource(DashboardServiceSource):
                         continue
                     yield Either(
                         right=CreateChartRequest(
-                            name=chart.id,
+                            name=EntityName(chart.id),
                             displayName=chart_display_name,
                             chartType=ChartType.Other.value,
-                            sourceUrl=self._get_chart_url(
-                                report_id=chart.reportId,
-                                workspace_id=self.context.get().workspace.id,
-                                dashboard_id=dashboard_details.id,
+                            sourceUrl=SourceUrl(
+                                self._get_chart_url(
+                                    report_id=chart.reportId,
+                                    workspace_id=self.context.get().workspace.id,
+                                    dashboard_id=dashboard_details.id,
+                                )
+                            ),
+                            service=FullyQualifiedEntityName(
+                                self.context.get().dashboard_service
                             ),
-                            service=self.context.get().dashboard_service,
                         )
                     )
                 except Exception as exc:

metadata/ingestion/source/dashboard/powerbi/models.py

@@ -15,6 +15,7 @@ from datetime import datetime
 from typing import List, Optional
 
 from pydantic import BaseModel, Field
+from typing_extensions import Annotated
 
 
 class Tile(BaseModel):
@@ -24,11 +25,11 @@ class Tile(BaseModel):
     """
 
     id: str
-    title: Optional[str]
-    subTitle: Optional[str]
-    embedUrl: Optional[str]
-    datasetId: Optional[str]
-    reportId: Optional[str]
+    title: Optional[str] = None
+    subTitle: Optional[str] = None
+    embedUrl: Optional[str] = None
+    datasetId: Optional[str] = None
+    reportId: Optional[str] = None
 
 
 class PowerBIDashboard(BaseModel):
@@ -39,8 +40,8 @@ class PowerBIDashboard(BaseModel):
 
     id: str
     displayName: str
-    webUrl: Optional[str]
-    embedUrl: Optional[str]
+    webUrl: Optional[str] = None
+    embedUrl: Optional[str] = None
     tiles: Optional[List[Tile]] = []
 
 
@@ -52,7 +53,7 @@ class PowerBIReport(BaseModel):
 
     id: str
     name: str
-    datasetId: Optional[str]
+    datasetId: Optional[str] = None
 
 
 class DashboardsResponse(BaseModel):
@@ -92,8 +93,8 @@ class PowerBiColumns(BaseModel):
     """
 
     name: str
-    dataType: Optional[str]
-    columnType: Optional[str]
+    dataType: Optional[str] = None
+    columnType: Optional[str] = None
 
 
 class PowerBiTable(BaseModel):
@@ -103,8 +104,8 @@ class PowerBiTable(BaseModel):
     """
 
     name: str
-    columns: Optional[List[PowerBiColumns]]
-    description: Optional[str]
+    columns: Optional[List[PowerBiColumns]] = None
+    description: Optional[str] = None
 
 
 class TablesResponse(BaseModel):
@@ -126,7 +127,7 @@ class Dataset(BaseModel):
     id: str
     name: str
     tables: Optional[List[PowerBiTable]] = []
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class DatasetResponse(BaseModel):
@@ -146,9 +147,9 @@ class Group(BaseModel):
     """
 
     id: str
-    name: Optional[str]
-    type: Optional[str]
-    state: Optional[str]
+    name: Optional[str] = None
+    type: Optional[str] = None
+    state: Optional[str] = None
     dashboards: Optional[List[PowerBIDashboard]] = []
     reports: Optional[List[PowerBIReport]] = []
     datasets: Optional[List[Dataset]] = []
@@ -173,7 +174,7 @@ class WorkSpaceScanResponse(BaseModel):
 
     id: str
     createdDateTime: datetime
-    status: Optional[str]
+    status: Optional[str] = None
 
 
 class Workspaces(BaseModel):
@@ -190,8 +191,8 @@ class PowerBiToken(BaseModel):
     PowerBI Token Model
     """
 
-    expires_in: Optional[int]
-    access_token: Optional[str]
+    expires_in: Optional[int] = None
+    access_token: Optional[str] = None
 
 
 class RemoteArtifacts(BaseModel):
@@ -208,7 +209,9 @@ class ConnectionFile(BaseModel):
     PowerBi Connection File Model
     """
 
-    RemoteArtifacts: Optional[List[RemoteArtifacts]]
+    RemoteArtifacts: Annotated[
+        Optional[List[RemoteArtifacts]], Field(None, description="Remote Artifacts")
+    ]
 
 
 class DataModelSchema(BaseModel):
@@ -216,5 +219,5 @@ class DataModelSchema(BaseModel):
     PowerBi Data Model Schema Model
     """
 
-    tables: Optional[List[PowerBiTable]]
-    connectionFile: Optional[ConnectionFile]
+    tables: Optional[List[PowerBiTable]] = None
+    connectionFile: Optional[ConnectionFile] = None

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -59,7 +59,7 @@ class QlikCloudClient:
         self.socket_connection = None
 
         client_config: ClientConfig = ClientConfig(
-            base_url=self.config.hostPort,
+            base_url=str(self.config.hostPort),
             api_version=API_VERSION,
             auth_header=AUTHORIZATION_HEADER,
             auth_token=lambda: (self.config.token.get_secret_value(), 0),
@@ -145,7 +145,7 @@ class QlikCloudClient:
         resp_apps = self.client.get("/v1/items?resourceType=app")
         if resp_apps:
             resp = QlikAppResponse(**resp_apps)
-            yield from resp.apps
+            return list(resp.apps)
 
     def get_dashboard_details(self, dashboard_id: str) -> Optional[QlikApp]:
         """

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -32,6 +32,12 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -60,8 +66,8 @@ class QlikcloudSource(QliksenseSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: QlikCloudConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: QlikCloudConnection = config.serviceConnection.root.config
         if not isinstance(connection, QlikCloudConnection):
             raise InvalidSourceException(
                 f"Expected QlikCloudConnection, but got {connection}"
@@ -117,22 +123,26 @@ class QlikcloudSource(QliksenseSource):
             dashboard_url = f"{clean_uri(self.service_connection.hostPort)}/sense/app/{dashboard_details.id}/overview"
 
             dashboard_request = CreateDashboardRequest(
-                name=dashboard_details.id,
-                sourceUrl=dashboard_url,
+                name=EntityName(dashboard_details.id),
+                sourceUrl=SourceUrl(dashboard_url),
                 displayName=dashboard_details.name,
-                description=dashboard_details.description,
+                description=Markdown(dashboard_details.description)
+                if dashboard_details.description
+                else None,
                 project=self.context.get().project_name,
                 charts=[
-                    fqn.build(
-                        self.metadata,
-                        entity_type=Chart,
-                        service_name=self.context.get().dashboard_service,
-                        chart_name=chart,
+                    FullyQualifiedEntityName(
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.get().dashboard_service,
+                            chart_name=chart,
+                        )
                     )
                     for chart in self.context.get().charts or []
                 ],
-                service=self.context.get().dashboard_service,
-                owner=self.get_owner_ref(dashboard_details=dashboard_details),
+                service=FullyQualifiedEntityName(self.context.get().dashboard_service),
+                owners=self.get_owner_ref(dashboard_details=dashboard_details),
             )
             yield Either(right=dashboard_request)
             self.register_record(dashboard_request=dashboard_request)
@@ -160,7 +170,7 @@ class QlikcloudSource(QliksenseSource):
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
-                    service_name=db_service_entity.name.__root__,
+                    service_name=db_service_entity.name.root,
                     schema_name=schema_name,
                     table_name=data_model_entity.displayName,
                     database_name=database_name,
@@ -231,11 +241,13 @@ class QlikcloudSource(QliksenseSource):
                     continue
                 yield Either(
                     right=CreateChartRequest(
-                        name=chart.qInfo.qId,
+                        name=EntityName(chart.qInfo.qId),
                         displayName=chart.qMeta.title,
-                        description=chart.qMeta.description,
+                        description=Markdown(chart.qMeta.description)
+                        if chart.qMeta.description
+                        else None,
                         chartType=ChartType.Other,
-                        sourceUrl=chart_url,
+                        sourceUrl=SourceUrl(chart_url),
                         service=self.context.get().dashboard_service,
                     )
                 )

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -20,11 +20,11 @@ from pydantic import BaseModel, Field
 class QlikApp(BaseModel):
     """QlikCloud App model"""
 
-    description: Optional[str]
-    name: Optional[str]
+    description: Optional[str] = None
+    name: Optional[str] = None
     id: str
-    app_id: Optional[str] = Field(alias="resourceId", default=None)
-    published: Optional[bool]
+    app_id: Optional[str] = Field(None, alias="resourceId")
+    published: Optional[bool] = None
 
 
 class QlikLink(BaseModel):

metadata/ingestion/source/dashboard/qliksense/client.py

@@ -78,9 +78,9 @@ class QlikSenseClient:
             return context
 
         self.ssl_manager = SSLManager(
-            ca=self.config.certificates.sslConfig.__root__.caCertificate,
-            cert=self.config.certificates.sslConfig.__root__.sslCertificate,
-            key=self.config.certificates.sslConfig.__root__.sslKey,
+            ca=self.config.certificates.sslConfig.root.caCertificate,
+            cert=self.config.certificates.sslConfig.root.sslCertificate,
+            key=self.config.certificates.sslConfig.root.sslKey,
         )
 
         return self.ssl_manager.setup_ssl(self.config)