openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/ingestion/source/database/deltalake/metadata.py

@@ -11,12 +11,8 @@
 """
 Deltalake source methods.
 """
-import re
 import traceback
-from enum import Enum
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
-
-from pyspark.sql.utils import AnalysisException, ParseException
+from typing import Any, Iterable, Optional, Tuple, Union
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
@@ -30,7 +26,7 @@ from metadata.generated.schema.api.data.createTable import CreateTableRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Table, TableType
+from metadata.generated.schema.entity.data.table import Table, TablePartition, TableType
 from metadata.generated.schema.entity.services.connections.database.deltaLakeConnection import (
     DeltaLakeConnection,
 )
@@ -43,40 +39,21 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection
-from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
 from metadata.ingestion.source.database.stored_procedures_mixin import QueryByProcedure
 from metadata.utils import fqn
-from metadata.utils.constants import DEFAULT_DATABASE
 from metadata.utils.filters import filter_by_schema, filter_by_table
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
 
-class SparkTableType(Enum):
-    MANAGED = "MANAGED"
-    TEMPORARY = "TEMPORARY"
-    VIEW = "VIEW"
-    EXTERNAL = "EXTERNAL"
-
-
-TABLE_TYPE_MAP = {
-    SparkTableType.MANAGED.value: TableType.Regular,
-    SparkTableType.VIEW.value: TableType.View,
-    SparkTableType.EXTERNAL.value: TableType.External,
-}
-
-
-ARRAY_CHILD_START_INDEX = 6
-ARRAY_CHILD_END_INDEX = -1
-
-
 class MetaStoreNotFoundException(Exception):
     """
     Metastore is not passed thorugh file or url
@@ -99,29 +76,22 @@ class DeltalakeSource(DatabaseServiceSource):
         self.source_config: DatabaseServiceMetadataPipeline = (
             self.config.sourceConfig.config
         )
+
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
-        self.spark = get_connection(self.service_connection)
-
-        self.table_type_map = {
-            TableType.External.value.lower(): TableType.External.value,
-            TableType.View.value.lower(): TableType.View.value,
-            TableType.SecureView.value.lower(): TableType.SecureView.value,
-            TableType.Iceberg.value.lower(): TableType.Iceberg.value,
-        }
-        self.array_datatype_replace_map = {"(": "<", ")": ">", "=": ":", "<>": ""}
-        self.table_constraints = None
-        self.database_source_state = set()
-
-        self.connection_obj = self.spark
+
+        self.service_connection = self.config.serviceConnection.root.config
+        self.connection = get_connection(self.service_connection)
+        self.client = self.connection.client
+
+        self.connection_obj = self.connection
         self.test_connection()
 
     @classmethod
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: DeltaLakeConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: DeltaLakeConnection = config.serviceConnection.root.config
         if not isinstance(connection, DeltaLakeConnection):
             raise InvalidSourceException(
                 f"Expected DeltaLakeConnection, but got {connection}"
@@ -138,7 +108,7 @@ class DeltalakeSource(DatabaseServiceSource):
         apply the necessary filters.
         """
 
-        yield self.service_connection.databaseName or DEFAULT_DATABASE
+        yield from self.client.get_database_names(self.service_connection)
 
     def yield_database(
         self, database_name: str
@@ -158,24 +128,23 @@ class DeltalakeSource(DatabaseServiceSource):
         """
         return schema names
         """
-        schemas = self.spark.catalog.listDatabases()
-        for schema in schemas:
+        for schema in self.client.get_database_schema_names(self.service_connection):
             schema_fqn = fqn.build(
                 self.metadata,
                 entity_type=DatabaseSchema,
                 service_name=self.context.get().database_service,
                 database_name=self.context.get().database,
-                schema_name=schema.name,
+                schema_name=schema,
             )
             if filter_by_schema(
                 self.config.sourceConfig.config.schemaFilterPattern,
                 schema_fqn
                 if self.config.sourceConfig.config.useFqnForFiltering
-                else schema.name,
+                else schema,
             ):
                 self.status.filter(schema_fqn, "Schema Filtered Out")
                 continue
-            yield schema.name
+            yield schema
 
     def yield_database_schema(
         self, schema_name: str
@@ -186,12 +155,14 @@ class DeltalakeSource(DatabaseServiceSource):
         """
         yield Either(
             right=CreateDatabaseSchemaRequest(
-                name=schema_name,
-                database=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
+                name=EntityName(schema_name),
+                database=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Database,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                    )
                 ),
             )
         )
@@ -206,53 +177,56 @@ class DeltalakeSource(DatabaseServiceSource):
         :return: tables or views, depending on config
         """
         schema_name = self.context.get().database_schema
-        for table in self.spark.catalog.listTables(dbName=schema_name):
+        for table_info in self.client.get_table_info(
+            self.service_connection, schema_name=schema_name
+        ):
             try:
-                table_name = table.name
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
                     service_name=self.context.get().database_service,
                     database_name=self.context.get().database,
                     schema_name=self.context.get().database_schema,
-                    table_name=table.name,
+                    table_name=table_info.name,
                 )
                 if filter_by_table(
                     self.source_config.tableFilterPattern,
-                    table_fqn if self.source_config.useFqnForFiltering else table.name,
+                    table_fqn
+                    if self.source_config.useFqnForFiltering
+                    else table_info.name,
                 ):
                     self.status.filter(
                         table_fqn,
                         "Table Filtered Out",
                     )
                     continue
+
                 if (
                     self.source_config.includeTables
-                    and table.tableType
-                    and table.tableType != SparkTableType.VIEW.value
+                    and table_info._type != TableType.View
                 ):
-                    # We will skip ingesting any TMP table
-                    if table.tableType == SparkTableType.TEMPORARY.value:
-                        logger.debug(f"Skipping temporary table {table.name}")
-                        continue
-
-                    self.context.get().table_description = table.description
-                    yield table_name, TABLE_TYPE_MAP.get(
-                        table.tableType, TableType.Regular
-                    )
+                    table_info = self.client.update_table_info(table_info)
+                    self.context.get().table_description = table_info.description
+                    self.context.get().table_columns = table_info.columns
+                    self.context.get().table_partitions = table_info.table_partitions
+                    yield table_info.name, table_info._type
 
                 if (
                     self.source_config.includeViews
-                    and table.tableType
-                    and table.tableType == SparkTableType.VIEW.value
+                    and table_info._type == TableType.View
                 ):
-                    self.context.get().table_description = table.description
-                    yield table_name, TableType.View
+                    table_info = self.client.update_table_info(table_info)
+                    self.context.get().table_description = table_info.description
+                    self.context.get().table_columns = table_info.columns
+                    self.context.get().table_partitions = table_info.table_partitions
+                    yield table_info.name, table_info._type
 
             except Exception as exc:
                 logger.debug(traceback.format_exc())
-                logger.warning(f"Unexpected exception for table [{table}]: {exc}")
-                self.status.warnings.append(f"{self.config.serviceName}.{table.name}")
+                logger.warning(f"Unexpected exception for table [{table_info}]: {exc}")
+                self.status.warnings.append(
+                    f"{self.config.serviceName}.{table_info.name}"
+                )
 
     def yield_table(
         self, table_name_and_type: Tuple[str, TableType]
@@ -263,28 +237,36 @@ class DeltalakeSource(DatabaseServiceSource):
         """
         table_name, table_type = table_name_and_type
         schema_name = self.context.get().database_schema
+
         try:
-            columns = self.get_columns(schema_name, table_name)
             view_definition = (
-                self._fetch_view_schema(table_name)
+                self.client.fetch_view_schema(table_name)
                 if table_type == TableType.View
                 else None
             )
 
+            table_partitions = self.context.get().table_partitions
+            table_partition = (
+                TablePartition(columns=table_partitions) if table_partitions else None
+            )
+
             table_request = CreateTableRequest(
-                name=table_name,
+                name=EntityName(table_name),
                 tableType=table_type,
                 description=self.context.get().table_description,
-                columns=columns,
+                columns=self.context.get().table_columns,
                 tableConstraints=None,
-                databaseSchema=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=DatabaseSchema,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                    schema_name=schema_name,
+                databaseSchema=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=DatabaseSchema,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                        schema_name=schema_name,
+                    )
                 ),
                 schemaDefinition=view_definition,
+                tablePartition=table_partition,
             )
 
             yield Either(right=table_request)
@@ -301,114 +283,6 @@ class DeltalakeSource(DatabaseServiceSource):
     def prepare(self):
         """Nothing to prepare"""
 
-    def _fetch_view_schema(self, view_name: str) -> Optional[Dict]:
-        try:
-            describe_output = self.spark.sql(f"describe extended {view_name}").collect()
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(
-                f"Unexpected exception to fetch view schema [{view_name}]: {exc}"
-            )
-            return None
-        view_detail = {}
-        col_details = False
-
-        for row in describe_output:
-            row_dict = row.asDict()
-            if col_details:
-                view_detail[row_dict["col_name"]] = row_dict["data_type"]
-            if "# Detailed Table" in row_dict["col_name"]:
-                col_details = True
-        return view_detail.get("View Text")
-
-    def _check_col_length(self, datatype, col_raw_type):
-        if datatype and datatype.upper() in {"CHAR", "VARCHAR", "BINARY", "VARBINARY"}:
-            try:
-                return col_raw_type.length if col_raw_type.length else 1
-            except AttributeError:
-                return 1
-        return None
-
-    def _get_display_data_type(self, row):
-        display_data_type = repr(row["data_type"]).lower()
-        for original, new in self.array_datatype_replace_map.items():
-            display_data_type = display_data_type.replace(original, new)
-        return display_data_type
-
-    def _get_col_info(self, row):
-        parsed_string = (
-            ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
-                row["data_type"]
-            )
-        )
-        if parsed_string:
-            parsed_string["dataLength"] = self._check_col_length(
-                parsed_string["dataType"], row["data_type"]
-            )
-            if row["data_type"] == "array":
-                array_data_type_display = self._get_display_data_type(row)
-                parsed_string["dataTypeDisplay"] = array_data_type_display
-                # Parse Primitive Datatype string
-                # if Datatype is Arrya(int) -> Parse int
-                parsed_string[
-                    "arrayDataType"
-                ] = ColumnTypeParser._parse_primitive_datatype_string(  # pylint: disable=protected-access
-                    array_data_type_display[
-                        ARRAY_CHILD_START_INDEX:ARRAY_CHILD_END_INDEX
-                    ]
-                )[
-                    "dataType"
-                ]
-            column = Column(name=row["col_name"], **parsed_string)
-        else:
-            col_type = re.search(r"^\w+", row["data_type"]).group(0)
-            charlen = re.search(r"\(([\d]+)\)", row["data_type"])
-            if charlen:
-                charlen = int(charlen.group(1))
-            if (
-                col_type.upper() in {"CHAR", "VARCHAR", "VARBINARY", "BINARY"}
-                and charlen is None
-            ):
-                charlen = 1
-
-            column = Column(
-                name=row["col_name"],
-                description=row.get("comment"),
-                dataType=col_type,
-                dataLength=charlen,
-                displayName=row["data_type"],
-            )
-        return column
-
-    def get_columns(self, schema: str, table: str) -> List[Column]:
-        """
-        Method to handle table columns
-        """
-        field_dict: Dict[str, Any] = {}
-        table_name = f"{schema}.{table}"
-        try:
-            raw_columns = self.spark.sql(f"describe {table_name}").collect()
-            for field in self.spark.table(f"{table_name}").schema:
-                field_dict[field.name] = field
-        except (AnalysisException, ParseException) as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(
-                f"Unexpected exception getting columns for [{table_name}]: {exc}"
-            )
-            return []
-        parsed_columns: List[Column] = []
-        partition_cols = False
-        for row in raw_columns:
-            col_name = row["col_name"]
-            if col_name == "" or "#" in col_name:
-                partition_cols = True
-                continue
-            if not partition_cols:
-                column = self._get_col_info(row)
-                parsed_columns.append(column)
-
-        return parsed_columns
-
     def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
         yield from []
 

metadata/ingestion/source/database/domodatabase/metadata.py

@@ -14,7 +14,7 @@ Domo Database source to extract metadata
 """
 
 import traceback
-from typing import Any, Iterable, List, Optional, Tuple, Union
+from typing import Any, Iterable, Optional, Tuple, Union
 
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
@@ -28,7 +28,12 @@ from metadata.generated.schema.api.data.createTable import CreateTableRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
-from metadata.generated.schema.entity.data.table import Column, Table, TableType
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    ColumnName,
+    Table,
+    TableType,
+)
 from metadata.generated.schema.entity.services.connections.database.domoDatabaseConnection import (
     DomoDatabaseConnection,
 )
@@ -41,7 +46,8 @@ from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -51,6 +57,7 @@ from metadata.ingestion.source.database.database_service import DatabaseServiceS
 from metadata.ingestion.source.database.domodatabase.models import (
     OutputDataset,
     Owner,
+    Schema,
     SchemaColumn,
     User,
 )
@@ -77,7 +84,7 @@ class DomodatabaseSource(DatabaseServiceSource):
             self.config.sourceConfig.config
         )
         self.metadata = metadata
-        self.service_connection = self.config.serviceConnection.__root__.config
+        self.service_connection = self.config.serviceConnection.root.config
         self.domo_client = get_connection(self.service_connection)
         self.connection_obj = self.domo_client
         self.test_connection()
@@ -89,8 +96,8 @@ class DomodatabaseSource(DatabaseServiceSource):
         metadata: OpenMetadata,
         pipeline_name: Optional[str] = None,
     ):
-        config = WorkflowSource.parse_obj(config_dict)
-        connection: DomoDatabaseConnection = config.serviceConnection.__root__.config
+        config = WorkflowSource.model_validate(config_dict)
+        connection: DomoDatabaseConnection = config.serviceConnection.root.config
         if not isinstance(connection, DomoDatabaseConnection):
             raise InvalidSourceException(
                 f"Expected DomoDatabaseConnection, but got {connection}"
@@ -106,7 +113,7 @@ class DomodatabaseSource(DatabaseServiceSource):
     ) -> Iterable[Either[CreateDatabaseRequest]]:
         yield Either(
             right=CreateDatabaseRequest(
-                name=database_name,
+                name=EntityName(database_name),
                 service=self.context.get().database_service,
             )
         )
@@ -120,12 +127,14 @@ class DomodatabaseSource(DatabaseServiceSource):
     ) -> Iterable[Either[CreateDatabaseSchemaRequest]]:
         yield Either(
             right=CreateDatabaseSchemaRequest(
-                name=schema_name,
-                database=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=Database,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
+                name=EntityName(schema_name),
+                database=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=Database,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                    )
                 ),
             )
         )
@@ -167,7 +176,7 @@ class DomodatabaseSource(DatabaseServiceSource):
                 )
             )
 
-    def get_owners(self, owner: Owner) -> Optional[EntityReference]:
+    def get_owners(self, owner: Owner) -> Optional[EntityReferenceList]:
         try:
             owner_details = User(**self.domo_client.users_get(owner.id))
             if owner_details.email:
@@ -177,31 +186,29 @@ class DomodatabaseSource(DatabaseServiceSource):
         return None
 
     def yield_table(
-        self, table_name_and_type: Tuple[str, str]
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[CreateTableRequest]]:
         table_id, table_type = table_name_and_type
         try:
             table_constraints = None
             table_object = OutputDataset(**self.domo_client.datasets.get(table_id))
-            columns = (
-                self.get_columns(table_object.schemas.columns)
-                if table_object.columns
-                else []
-            )
+            columns = self.get_columns(table_object)
             table_request = CreateTableRequest(
-                name=table_object.name,
+                name=EntityName(table_object.name),
                 displayName=table_object.name,
                 tableType=table_type,
                 description=table_object.description,
                 columns=columns,
-                owner=self.get_owners(owner=table_object.owner),
+                owners=self.get_owners(owner=table_object.owner),
                 tableConstraints=table_constraints,
-                databaseSchema=fqn.build(
-                    metadata=self.metadata,
-                    entity_type=DatabaseSchema,
-                    service_name=self.context.get().database_service,
-                    database_name=self.context.get().database,
-                    schema_name=self.context.get().database_schema,
+                databaseSchema=FullyQualifiedEntityName(
+                    fqn.build(
+                        metadata=self.metadata,
+                        entity_type=DatabaseSchema,
+                        service_name=self.context.get().database_service,
+                        database_name=self.context.get().database,
+                        schema_name=self.context.get().database_schema,
+                    )
                 ),
                 sourceUrl=self.get_source_url(
                     table_name=table_id,
@@ -218,19 +225,57 @@ class DomodatabaseSource(DatabaseServiceSource):
                 )
             )
 
-    def get_columns(self, table_object: List[SchemaColumn]):
+    def get_columns_from_federated_dataset(self, table_name: str, dataset_id: str):
+        """
+        Method to retrieve the column metadata from federated datasets
+        """
+        try:
+            # SQL query to get all columns without fetching any rows
+            sql_query = f'SELECT * FROM "{table_name}" LIMIT 1'
+            schema_columns = []
+            response = self.domo_client.datasets.query(dataset_id, sql_query)
+            if response:
+                for i, column_name in enumerate(response["columns"] or []):
+                    schema_column = SchemaColumn(
+                        name=column_name, type=response["metadata"][i]["type"]
+                    )
+                    schema_columns.append(schema_column)
+            if schema_columns:
+                return Schema(columns=schema_columns)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error while fetching columns from federated dataset {table_name} - {exc}"
+            )
+        return None
+
+    def get_columns(self, table_object: OutputDataset):
+        """
+        Method to get domo table columns
+        """
         row_order = 1
         columns = []
-        for column in table_object:
-            columns.append(
-                Column(
-                    name=column.name,
-                    description=column.description,
-                    dataType=column.type,
-                    ordinalPosition=row_order,
-                )
+        if not table_object.schemas or not table_object.schemas.columns:
+            table_object.schemas = self.get_columns_from_federated_dataset(
+                table_name=table_object.name, dataset_id=table_object.id
             )
-            row_order += 1
+
+        for column in table_object.schemas.columns or []:
+            try:
+                columns.append(
+                    Column(
+                        name=ColumnName(column.name),
+                        description=column.description,
+                        dataType=column.type,
+                        ordinalPosition=row_order,
+                    )
+                )
+                row_order += 1
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Error while fetching details of column {column} - {exc}"
+                )
         return columns
 
     def yield_tag(

metadata/ingestion/source/database/domodatabase/models.py

@@ -15,18 +15,18 @@ Domo Database Source Model module
 
 from typing import List, Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class DomoDatabaseBaseModel(BaseModel):
-    class Config:
-        extra = Extra.allow
+    model_config = ConfigDict(extra="allow")
 
     id: str
     name: str
 
 
 class User(DomoDatabaseBaseModel):
+    id: int
     email: str
     role: str
 
@@ -34,7 +34,7 @@ class User(DomoDatabaseBaseModel):
 class SchemaColumn(BaseModel):
     type: str
     name: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class Schema(BaseModel):
@@ -49,6 +49,6 @@ class Owner(DomoDatabaseBaseModel):
 class OutputDataset(DomoDatabaseBaseModel):
     rows: int
     columns: int
-    schemas: Optional[Schema] = Field(alias="schema")
+    schemas: Optional[Schema] = Field(None, alias="schema")
     owner: Owner
-    description: Optional[str]
+    description: Optional[str] = None

metadata/ingestion/source/database/doris/metadata.py

@@ -147,7 +147,7 @@ class DorisSource(CommonDbSourceService):
 
     def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         self.ssl_manager = None
-        service_connection = config.serviceConnection.__root__.config
+        service_connection = config.serviceConnection.root.config
         self.ssl_manager: SSLManager = check_ssl_and_init(service_connection)
         if self.ssl_manager:
             service_connection = self.ssl_manager.setup_ssl(service_connection)
@@ -157,10 +157,10 @@ class DorisSource(CommonDbSourceService):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
         if config.serviceConnection is None:
             raise InvalidSourceException("Missing service connection")
-        connection = cast(DorisConnection, config.serviceConnection.__root__.config)
+        connection = cast(DorisConnection, config.serviceConnection.root.config)
         if not isinstance(connection, DorisConnection):
             raise InvalidSourceException(
                 f"Expected DorisConnection, but got {connection}"