openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/database/athena/utils.py

@@ -0,0 +1,191 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Athena utils module"""
+
+from copy import deepcopy
+from typing import Dict, List, Optional
+
+from pyathena.sqlalchemy.util import _HashableDict
+from sqlalchemy import types
+from sqlalchemy.engine import reflection
+
+from metadata.ingestion.source import sqa_types
+from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
+from metadata.utils.sqlalchemy_utils import is_complex_type
+
+
+@reflection.cache
+def _get_column_type(self, type_):
+    """
+    Function overwritten from AthenaDialect
+    to add custom SQA typing.
+    """
+    type_ = type_.replace(" ", "").lower()
+    match = self._pattern_column_type.match(type_)  # pylint: disable=protected-access
+    if match:
+        name = match.group(1).lower()
+        length = match.group(2)
+    else:
+        name = type_.lower()
+        length = None
+
+    args = []
+    col_map = {
+        "boolean": types.BOOLEAN,
+        "float": types.FLOAT,
+        "double": types.FLOAT,
+        "real": types.FLOAT,
+        "tinyint": types.INTEGER,
+        "smallint": types.INTEGER,
+        "integer": types.INTEGER,
+        "int": types.INTEGER,
+        "bigint": types.BIGINT,
+        "string": types.String,
+        "date": types.DATE,
+        "timestamp": types.TIMESTAMP,
+        "binary": types.BINARY,
+        "varbinary": types.BINARY,
+        "array": types.ARRAY,
+        "json": types.JSON,
+        "struct": sqa_types.SQAStruct,
+        "row": sqa_types.SQAStruct,
+        "map": sqa_types.SQAMap,
+        "decimal": types.DECIMAL,
+        "varchar": types.VARCHAR,
+        "char": types.CHAR,
+    }
+    if name in ["decimal", "char", "varchar"]:
+        col_type = col_map[name]
+        if length:
+            args = [int(l) for l in length.split(",")]
+    elif type_.startswith("array"):
+        parsed_type = (
+            ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
+                type_
+            )
+        )
+        col_type = col_map["array"]
+        if parsed_type["arrayDataType"].lower().startswith("array"):
+            # as OpenMetadata doesn't store any details on children of array, we put
+            # in type as string as default to avoid Array item_type required issue
+            # from sqlalchemy types
+            args = [types.String]
+        else:
+            args = [col_map.get(parsed_type.get("arrayDataType").lower(), types.String)]
+    elif col_map.get(name):
+        col_type = col_map.get(name)
+    else:
+        logger.warning(f"Did not recognize type '{type_}'")
+        col_type = types.NullType
+    return col_type(*args)
+
+
+# pylint: disable=unused-argument
+def _get_projection_details(
+    columns: List[Dict], projection_parameters: Dict
+) -> List[Dict]:
+    """Get the projection details for the columns
+
+    Args:
+        columns (List[Dict]): list of columns
+        projection_parameters (Dict): projection parameters
+    """
+    if not projection_parameters:
+        return columns
+
+    columns = deepcopy(columns)
+    for col in columns:
+        projection_details = next(
+            ({k: v} for k, v in projection_parameters.items() if k == col["name"]), None
+        )
+        if projection_details:
+            col["projection_type"] = projection_details[col["name"]]
+
+    return columns
+
+
+@reflection.cache
+def get_columns(self, connection, table_name, schema=None, **kw):
+    """
+    Method to handle table columns
+    """
+    metadata = self._get_table(  # pylint: disable=protected-access
+        connection, table_name, schema=schema, **kw
+    )
+    columns = [
+        {
+            "name": c.name,
+            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "nullable": True,
+            "default": None,
+            "autoincrement": False,
+            "comment": c.comment,
+            "system_data_type": c.type,
+            "is_complex": is_complex_type(c.type),
+            "dialect_options": {"awsathena_partition": True},
+        }
+        for c in metadata.partition_keys
+    ]
+
+    if kw.get("only_partition_columns"):
+        # Return projected partition information to set partition type in `get_table_partition_details`
+        # projected partition fields are stored in the form of `projection.<field_name>.type` as a table parameter
+        projection_parameters = {
+            key_.split(".")[1]: value_
+            for key_, value_ in metadata.parameters.items()
+            if key_.startswith("projection") and key_.endswith("type")
+        }
+        columns = _get_projection_details(columns, projection_parameters)
+        return columns
+
+    columns += [
+        {
+            "name": c.name,
+            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "nullable": True,
+            "default": None,
+            "autoincrement": False,
+            "comment": c.comment,
+            "system_data_type": c.type,
+            "is_complex": is_complex_type(c.type),
+            "dialect_options": {"awsathena_partition": None},
+        }
+        for c in metadata.columns
+    ]
+
+    return columns
+
+
+@reflection.cache
+def get_view_definition(self, connection, view_name, schema=None, **kw):
+    """
+    Gets the view definition
+    """
+    full_view_name = f'"{view_name}"' if not schema else f'"{schema}"."{view_name}"'
+    res = connection.execute(f"SHOW CREATE VIEW {full_view_name}").fetchall()
+    if res:
+        return "\n".join(i[0] for i in res)
+    return None
+
+
+def get_table_options(
+    self, connection: "Connection", table_name: str, schema: Optional[str] = None, **kw
+):
+    metadata = self._get_table(connection, table_name, schema=schema, **kw)
+    return {
+        "awsathena_location": metadata.location,
+        "awsathena_compression": metadata.compression,
+        "awsathena_row_format": metadata.row_format,
+        "awsathena_file_format": metadata.file_format,
+        "awsathena_serdeproperties": _HashableDict(metadata.serde_properties),
+        "awsathena_tblproperties": _HashableDict(metadata.table_properties),
+    }

metadata/ingestion/source/database/azuresql/metadata.py

@@ -62,8 +62,8 @@ class AzuresqlSource(CommonDbSourceService, MultiDBSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: AzureSQLConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: AzureSQLConnection = config.serviceConnection.root.config
         if not isinstance(connection, AzureSQLConnection):
             raise InvalidSourceException(
                 f"Expected AzureSQLConnection, but got {connection}"
@@ -79,8 +79,8 @@ class AzuresqlSource(CommonDbSourceService, MultiDBSource):
         yield from self._execute_database_query(AZURE_SQL_GET_DATABASES)
 
     def get_database_names(self) -> Iterable[str]:
-        if not self.config.serviceConnection.__root__.config.ingestAllDatabases:
-            configured_db = self.config.serviceConnection.__root__.config.database
+        if not self.config.serviceConnection.root.config.ingestAllDatabases:
+            configured_db = self.config.serviceConnection.root.config.database
             self.set_inspector(database_name=configured_db)
             yield configured_db
         else:

metadata/ingestion/source/database/azuresql/query_parser.py

@@ -37,8 +37,8 @@ class AzuresqlQueryParserSource(QueryParserSource, ABC):
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
         """Create class instance"""
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: AzureSQLConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: AzureSQLConnection = config.serviceConnection.root.config
         if not isinstance(connection, AzureSQLConnection):
             raise InvalidSourceException(
                 f"Expected Azuresql Connection, but got {connection}"

metadata/ingestion/source/database/bigquery/connection.py

@@ -26,6 +26,9 @@ from metadata.generated.schema.entity.automations.workflow import (
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
+from metadata.generated.schema.security.credentials.gcpCredentials import (
+    GcpCredentialsPath,
+)
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
     MultipleProjectId,
@@ -59,23 +62,32 @@ def get_connection_url(connection: BigQueryConnection) -> str:
         if isinstance(  # pylint: disable=no-else-return
             connection.credentials.gcpConfig.projectId, SingleProjectId
         ):
-            if not connection.credentials.gcpConfig.projectId.__root__:
+            if not connection.credentials.gcpConfig.projectId.root:
                 return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId or ''}"
             if (
                 not connection.credentials.gcpConfig.privateKey
-                and connection.credentials.gcpConfig.projectId.__root__
+                and connection.credentials.gcpConfig.projectId.root
             ):
-                project_id = connection.credentials.gcpConfig.projectId.__root__
+                project_id = connection.credentials.gcpConfig.projectId.root
                 os.environ["GOOGLE_CLOUD_PROJECT"] = project_id
-            return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.__root__}"
+            return f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId.root}"
         elif isinstance(connection.credentials.gcpConfig.projectId, MultipleProjectId):
-            for project_id in connection.credentials.gcpConfig.projectId.__root__:
+            for project_id in connection.credentials.gcpConfig.projectId.root:
                 if not connection.credentials.gcpConfig.privateKey and project_id:
                     # Setting environment variable based on project id given by user / set in ADC
                     os.environ["GOOGLE_CLOUD_PROJECT"] = project_id
                 return f"{connection.scheme.value}://{project_id}"
             return f"{connection.scheme.value}://"
 
+    # If gcpConfig is the JSON key path and projectId is defined, we use it by default
+    elif (
+        isinstance(connection.credentials.gcpConfig, GcpCredentialsPath)
+        and connection.credentials.gcpConfig.projectId
+    ):
+        return (
+            f"{connection.scheme.value}://{connection.credentials.gcpConfig.projectId}"
+        )
+
     return f"{connection.scheme.value}://"
 
 

metadata/ingestion/source/database/bigquery/helper.py

@@ -56,7 +56,7 @@ def get_inspector_details(
     kwargs = {}
     if isinstance(service_connection.credentials.gcpConfig, GcpCredentialsValues):
         service_connection.credentials.gcpConfig.projectId = SingleProjectId(
-            __root__=database_name
+            database_name
         )
         if service_connection.credentials.gcpImpersonateServiceAccount:
             kwargs[

metadata/ingestion/source/database/bigquery/metadata.py

@@ -11,6 +11,7 @@
 """
 Bigquery source module
 """
+import ast
 import os
 import traceback
 from typing import Dict, Iterable, List, Optional, Tuple
@@ -51,7 +52,11 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
 )
-from metadata.generated.schema.type.basic import EntityName, SourceUrl
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    SourceUrl,
+)
 from metadata.generated.schema.type.tagLabel import TagLabel
 from metadata.ingestion.api.delete import delete_entity_by_name
 from metadata.ingestion.api.models import Either
@@ -259,8 +264,8 @@ class BigquerySource(
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: BigQueryConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: BigQueryConnection = config.serviceConnection.root.config
         if not isinstance(connection, BigQueryConnection):
             raise InvalidSourceException(
                 f"Expected BigQueryConnection, but got {connection}"
@@ -355,11 +360,7 @@ class BigquerySource(
     def yield_tag(
         self, schema_name: str
     ) -> Iterable[Either[OMetaTagAndClassification]]:
-        """
-        Build tag context
-        :param _:
-        :return:
-        """
+        """Build tag context"""
         try:
             # Fetching labels on the databaseSchema ( dataset ) level
             dataset_obj = self.client.get_dataset(schema_name)
@@ -369,7 +370,7 @@ class BigquerySource(
                         tags=[value],
                         classification_name=key,
                         tag_description="Bigquery Dataset Label",
-                        classification_description="",
+                        classification_description="BigQuery Dataset Classification",
                         include_tags=self.source_config.includeTags,
                     )
             # Fetching policy tags on the column level
@@ -389,7 +390,7 @@ class BigquerySource(
                         tags=[tag.display_name for tag in policy_tags],
                         classification_name=taxonomy.display_name,
                         tag_description="Bigquery Policy Tag",
-                        classification_description="",
+                        classification_description="BigQuery Policy Classification",
                         include_tags=self.source_config.includeTags,
                     )
         except Exception as exc:
@@ -412,7 +413,10 @@ class BigquerySource(
             )
 
             query_result = [result.schema_description for result in query_resp.result()]
-            return fqn.unquote_name(query_result[0])
+
+            return str(
+                ast.literal_eval(query_result[0])
+            )  # To safely evaluate the string, unquote and interpret escaped characters
         except IndexError:
             logger.debug(f"No dataset description found for {schema_name}")
         except Exception as err:
@@ -484,12 +488,14 @@ class BigquerySource(
         """
 
         database_schema_request_obj = CreateDatabaseSchemaRequest(
-            name=schema_name,
-            database=fqn.build(
-                metadata=self.metadata,
-                entity_type=Database,
-                service_name=self.context.get().database_service,
-                database_name=self.context.get().database,
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                )
             ),
             description=self.get_schema_description(schema_name),
             sourceUrl=self.get_source_url(
@@ -526,7 +532,7 @@ class BigquerySource(
                     tags=[value],
                     classification_name=key,
                     tag_description="Bigquery Table Label",
-                    classification_description="",
+                    classification_description="BigQuery Table Classification",
                     include_tags=self.source_config.includeTags,
                 )
 
@@ -630,7 +636,7 @@ class BigquerySource(
                 self.connection, table_name, schema_name
             )
             schema_definition = (
-                str(schema_definition.strip())
+                str(schema_definition).strip()
                 if schema_definition is not None
                 else None
             )
@@ -639,49 +645,82 @@ class BigquerySource(
             logger.warning("Schema definition not implemented")
         return None
 
+    def _get_partition_column_name(
+        self, columns: List[Dict], partition_field_name: str
+    ):
+        """
+        Method to get the correct partition column name
+        """
+        try:
+            for column in columns or []:
+                column_name = column.get("name")
+                if column_name and (
+                    column_name.lower() == partition_field_name.lower()
+                ):
+                    return column_name
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error getting partition column name for {partition_field_name}: {exc}"
+            )
+        return None
+
     def get_table_partition_details(
         self, table_name: str, schema_name: str, inspector: Inspector
     ) -> Tuple[bool, Optional[TablePartition]]:
         """
         check if the table is partitioned table and return the partition details
         """
-        database = self.context.get().database
-        table = self.client.get_table(fqn._build(database, schema_name, table_name))
-        if table.time_partitioning is not None:
-            if table.time_partitioning.field:
-                table_partition = TablePartition(
+        try:
+            database = self.context.get().database
+            table = self.client.get_table(fqn._build(database, schema_name, table_name))
+            columns = inspector.get_columns(table_name, schema_name, db_name=database)
+            if table.time_partitioning is not None:
+                if table.time_partitioning.field:
+                    table_partition = TablePartition(
+                        columns=[
+                            PartitionColumnDetails(
+                                columnName=self._get_partition_column_name(
+                                    columns=columns,
+                                    partition_field_name=table.time_partitioning.field,
+                                ),
+                                interval=str(table.time_partitioning.type_),
+                                intervalType=PartitionIntervalTypes.TIME_UNIT,
+                            )
+                        ]
+                    )
+                    return True, table_partition
+                return True, TablePartition(
                     columns=[
                         PartitionColumnDetails(
-                            columnName=table.time_partitioning.field,
+                            columnName="_PARTITIONTIME"
+                            if table.time_partitioning.type_ == "HOUR"
+                            else "_PARTITIONDATE",
                             interval=str(table.time_partitioning.type_),
-                            intervalType=PartitionIntervalTypes.TIME_UNIT,
+                            intervalType=PartitionIntervalTypes.INGESTION_TIME,
                         )
                     ]
                 )
-                return True, table_partition
-            return True, TablePartition(
-                columns=[
-                    PartitionColumnDetails(
-                        columnName="_PARTITIONTIME"
-                        if table.time_partitioning.type_ == "HOUR"
-                        else "_PARTITIONDATE",
-                        interval=str(table.time_partitioning.type_),
-                        intervalType=PartitionIntervalTypes.INGESTION_TIME,
-                    )
-                ]
-            )
-        if table.range_partitioning:
-            table_partition = PartitionColumnDetails(
-                columnName=table.range_partitioning.field,
-                intervalType=PartitionIntervalTypes.INTEGER_RANGE,
-                interval=None,
+            if table.range_partitioning:
+                table_partition = PartitionColumnDetails(
+                    columnName=self._get_partition_column_name(
+                        columns=columns,
+                        partition_field_name=table.range_partitioning.field,
+                    ),
+                    intervalType=PartitionIntervalTypes.INTEGER_RANGE,
+                    interval=None,
+                )
+                if hasattr(table.range_partitioning, "range_") and hasattr(
+                    table.range_partitioning.range_, "interval"
+                ):
+                    table_partition.interval = table.range_partitioning.range_.interval
+                table_partition.columnName = table.range_partitioning.field
+                return True, TablePartition(columns=[table_partition])
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error getting table partition details for {table_name}: {exc}"
             )
-            if hasattr(table.range_partitioning, "range_") and hasattr(
-                table.range_partitioning.range_, "interval"
-            ):
-                table_partition.interval = table.range_partitioning.range_.interval
-            table_partition.columnName = table.range_partitioning.field
-            return True, TablePartition(columns=[table_partition])
         return False, None
 
     def clean_raw_data_type(self, raw_data_type):
@@ -767,7 +806,7 @@ class BigquerySource(
                 )
             ).all()
             for row in results:
-                stored_procedure = BigQueryStoredProcedure.parse_obj(dict(row))
+                stored_procedure = BigQueryStoredProcedure.model_validate(dict(row))
                 yield stored_procedure
 
     def yield_stored_procedure(
@@ -777,7 +816,7 @@ class BigquerySource(
 
         try:
             stored_procedure_request = CreateStoredProcedureRequest(
-                name=EntityName(__root__=stored_procedure.name),
+                name=EntityName(stored_procedure.name),
                 storedProcedureCode=StoredProcedureCode(
                     language=STORED_PROC_LANGUAGE_MAP.get(
                         stored_procedure.language or "SQL",
@@ -792,7 +831,7 @@ class BigquerySource(
                     schema_name=self.context.get().database_schema,
                 ),
                 sourceUrl=SourceUrl(
-                    __root__=self.get_stored_procedure_url(
+                    self.get_stored_procedure_url(
                         database_name=self.context.get().database,
                         schema_name=self.context.get().database_schema,
                         # Follow the same building strategy as tables

metadata/ingestion/source/database/bigquery/query_parser.py

@@ -48,8 +48,8 @@ class BigqueryQueryParserSource(QueryParserSource, ABC):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: BigQueryConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: BigQueryConnection = config.serviceConnection.root.config
         if not isinstance(connection, BigQueryConnection):
             raise InvalidSourceException(
                 f"Expected BigQueryConnection, but got {connection}"
@@ -82,7 +82,7 @@ class BigqueryQueryParserSource(QueryParserSource, ABC):
             project_ids = deepcopy(
                 self.service_connection.credentials.gcpConfig.projectId
             )
-            for project_id in project_ids.__root__:
+            for project_id in project_ids.root:
                 inspector_details = get_inspector_details(
                     project_id, self.service_connection
                 )

metadata/ingestion/source/database/bigtable/connection.py

@@ -40,9 +40,9 @@ def get_connection(connection: BigTableConnection):
     project_ids = None
     if isinstance(connection.credentials.gcpConfig, GcpCredentialsValues):
         project_ids = (
-            [connection.credentials.gcpConfig.projectId.__root__]
+            [connection.credentials.gcpConfig.projectId.root]
             if isinstance(connection.credentials.gcpConfig.projectId, SingleProjectId)
-            else connection.credentials.gcpConfig.projectId.__root__
+            else connection.credentials.gcpConfig.projectId.root
         )
     # admin=True is required to list instances and tables
     return MultiProjectClient(client_class=Client, project_ids=project_ids, admin=True)

metadata/ingestion/source/database/bigtable/metadata.py

@@ -74,8 +74,8 @@ class BigtableSource(CommonNoSQLSource, MultiDBSource):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: BigTableConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: BigTableConnection = config.serviceConnection.root.config
         if not isinstance(connection, BigTableConnection):
             raise InvalidSourceException(
                 f"Expected BigTableConnection, but got {connection}"

metadata/ingestion/source/database/clickhouse/connection.py

@@ -47,13 +47,13 @@ def get_connection(connection: ClickhouseConnection) -> Engine:
         if not connection.connectionArguments:
             connection.connectionArguments = init_empty_connection_arguments()
         if connection.secure:
-            connection.connectionArguments.__root__["secure"] = connection.secure
+            connection.connectionArguments.root["secure"] = connection.secure
         if connection.keyfile:
-            connection.connectionArguments.__root__["keyfile"] = connection.keyfile
+            connection.connectionArguments.root["keyfile"] = connection.keyfile
     if connection.https:
         if not connection.connectionOptions:
             connection.connectionOptions = init_empty_connection_options()
-        connection.connectionOptions.__root__["protocol"] = HTTPS_PROTOCOL
+        connection.connectionOptions.root["protocol"] = HTTPS_PROTOCOL
 
     return create_generic_db_connection(
         connection=connection,

metadata/ingestion/source/database/clickhouse/metadata.py

@@ -111,8 +111,8 @@ class ClickhouseSource(CommonDbSourceService):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: ClickhouseConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ClickhouseConnection = config.serviceConnection.root.config
         if not isinstance(connection, ClickhouseConnection):
             raise InvalidSourceException(
                 f"Expected ClickhouseConnection, but got {connection}"
@@ -156,16 +156,19 @@ class ClickhouseSource(CommonDbSourceService):
             if table_type in {TableType.View, TableType.MaterializedView}:
                 definition_fn = inspector.get_view_definition
                 schema_definition = definition_fn(table_name, schema_name)
-            else:
-                schema_definition = inspector.get_table_ddl(
-                    self.connection, table_name, schema_name
+                return (
+                    str(schema_definition).strip()
+                    if schema_definition is not None
+                    else None
                 )
-            schema_definition = (
-                str(schema_definition.strip())
+            schema_definition = inspector.get_table_ddl(
+                self.connection, table_name, schema_name
+            )
+            return (
+                str(schema_definition).strip()
                 if schema_definition is not None
                 else None
             )
-            return schema_definition
 
         except NotImplementedError:
             logger.warning("Schema definition not implemented")

metadata/ingestion/source/database/clickhouse/query_parser.py

@@ -42,8 +42,8 @@ class ClickhouseQueryParserSource(QueryParserSource, ABC):
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
-        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
-        connection: ClickhouseConnection = config.serviceConnection.__root__.config
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: ClickhouseConnection = config.serviceConnection.root.config
         if not isinstance(connection, ClickhouseConnection):
             raise InvalidSourceException(
                 f"Expected ClickhouseConnection, but got {connection}"
@@ -91,9 +91,9 @@ class ClickhouseQueryParserSource(QueryParserSource, ABC):
         schema_name_list = []
 
         for database in databases:
-            database_name_list.append(database.name.__root__)
+            database_name_list.append(database.name.root)
             if self.schema_field and database.databaseSchemas:
-                for schema in database.databaseSchemas.__root__:
+                for schema in database.databaseSchemas.root:
                     schema_name_list.append(schema.name)
 
         if self.schema_field and schema_name_list: