openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/profiler/interface/profiler_interface_factory.py

@@ -13,7 +13,8 @@
 Factory class for creating profiler interface objects
 """
 
-from typing import cast
+import importlib
+from typing import Dict, cast
 
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
@@ -50,63 +51,57 @@ from metadata.generated.schema.entity.services.connections.database.unityCatalog
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
 from metadata.profiler.factory import Factory
-from metadata.profiler.interface.nosql.profiler_interface import NoSQLProfilerInterface
-from metadata.profiler.interface.pandas.profiler_interface import (
-    PandasProfilerInterface,
-)
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
-from metadata.profiler.interface.sqlalchemy.bigquery.profiler_interface import (
-    BigQueryProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.databricks.profiler_interface import (
-    DatabricksProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.db2.profiler_interface import (
-    DB2ProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.mariadb.profiler_interface import (
-    MariaDBProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.profiler_interface import (
-    SQAProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.single_store.profiler_interface import (
-    SingleStoreProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.snowflake.profiler_interface import (
-    SnowflakeProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.trino.profiler_interface import (
-    TrinoProfilerInterface,
-)
-from metadata.profiler.interface.sqlalchemy.unity_catalog.profiler_interface import (
-    UnityCatalogProfilerInterface,
-)
 
 
 class ProfilerInterfaceFactory(Factory):
     def create(self, interface_type: str, *args, **kwargs):
         """Create interface object based on interface type"""
-        interface_class = self._interface_type.get(interface_type)
-        if not interface_class:
-            interface_class = self._interface_type.get(DatabaseConnection.__name__)
-        interface_class = cast(ProfilerInterface, interface_class)
-        return interface_class.create(*args, **kwargs)
+        interface_class_path = profiler_class_mapping.get(
+            interface_type, profiler_class_mapping[DatabaseConnection.__name__]
+        )
+        try:
+            module_path, class_name = interface_class_path.rsplit(".", 1)
+            module = importlib.import_module(module_path)
+            profiler_class = getattr(module, class_name)
+        except (ImportError, AttributeError) as e:
+            raise ImportError(f"Error importing {class_name} from {module_path}: {e}")
+        profiler_class = cast(ProfilerInterface, profiler_class)
+        return profiler_class.create(*args, **kwargs)
 
 
 profiler_interface_factory = ProfilerInterfaceFactory()
-profilers = {
-    DatabaseConnection.__name__: SQAProfilerInterface,
-    BigQueryConnection.__name__: BigQueryProfilerInterface,
-    SingleStoreConnection.__name__: SingleStoreProfilerInterface,
-    DatalakeConnection.__name__: PandasProfilerInterface,
-    MariaDBConnection.__name__: MariaDBProfilerInterface,
-    SnowflakeConnection.__name__: SnowflakeProfilerInterface,
-    TrinoConnection.__name__: TrinoProfilerInterface,
-    UnityCatalogConnection.__name__: UnityCatalogProfilerInterface,
-    DatabricksConnection.__name__: DatabricksProfilerInterface,
-    Db2Connection.__name__: DB2ProfilerInterface,
-    MongoDBConnection.__name__: NoSQLProfilerInterface,
-    DynamoDBConnection.__name__: NoSQLProfilerInterface,
+
+BASE_PROFILER_PATH = "metadata.profiler.interface"
+SQLALCHEMY_PROFILER_PATH = f"{BASE_PROFILER_PATH}.sqlalchemy"
+NOSQL_PROFILER_PATH = (
+    f"{BASE_PROFILER_PATH}.nosql.profiler_interface.NoSQLProfilerInterface"
+)
+PANDAS_PROFILER_PATH = (
+    f"{BASE_PROFILER_PATH}.pandas.profiler_interface.PandasProfilerInterface"
+)
+
+# Configuration for dynamic imports
+profiler_class_mapping: Dict[str, str] = {
+    DatabaseConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".profiler_interface.SQAProfilerInterface",
+    BigQueryConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".bigquery.profiler_interface.BigQueryProfilerInterface",
+    SingleStoreConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".single_store.profiler_interface.SingleStoreProfilerInterface",
+    DatalakeConnection.__name__: PANDAS_PROFILER_PATH,
+    MariaDBConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".mariadb.profiler_interface.MariaDBProfilerInterface",
+    SnowflakeConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".snowflake.profiler_interface.SnowflakeProfilerInterface",
+    TrinoConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".trino.profiler_interface.TrinoProfilerInterface",
+    UnityCatalogConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".unity_catalog.profiler_interface.UnityCatalogProfilerInterface",
+    DatabricksConnection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".databricks.profiler_interface.DatabricksProfilerInterface",
+    Db2Connection.__name__: SQLALCHEMY_PROFILER_PATH
+    + ".db2.profiler_interface.DB2ProfilerInterface",
+    MongoDBConnection.__name__: NOSQL_PROFILER_PATH,
+    DynamoDBConnection.__name__: NOSQL_PROFILER_PATH,
 }
-profiler_interface_factory.register_many(profilers)

metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py

@@ -36,15 +36,17 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
         result = super(  # pylint: disable=bad-super-call
             HiveCompiler, self
         ).visit_column(*args, **kwargs)
-        dot_count = result.count(".")
         # Here the databricks uses HiveCompiler.
         # the `result` here would be `db.schema.table` or `db.schema.table.column`
         # for struct it will be `db.schema.table.column.nestedchild.nestedchild` etc
         # the logic is to add the backticks to nested children.
-        if dot_count > 2:
-            splitted_result = result.split(".", 2)[-1].split(".")
-            result = ".".join(result.split(".", 2)[:-1])
-            result += "." + "`.`".join(splitted_result)
+        dot_count = result.count(".")
+        if dot_count > 1 and "." in result.split("`.`")[-1]:
+            splitted_result = result.split("`.")[-1].split(".")
+            result = "`.".join(result.split("`.")[:-1])
+            if result:
+                result += "`."
+            result += "`.`".join(splitted_result)
         return result
 
     def __init__(self, service_connection_config, **kwargs):
@@ -56,31 +58,40 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
         """Get struct columns"""
 
         columns_list = []
-        for idx, col in enumerate(columns):
+        for col in columns:
             if col.dataType != DataType.STRUCT:
-                col.name = ColumnName(__root__=f"{parent}.{col.name.__root__}")
-                col = build_orm_col(idx, col, DatabaseServiceType.Databricks)
-                col._set_parent(  # pylint: disable=protected-access
+                # For DBX struct we need to quote the column name as `a`.`b`.`c`
+                # otherwise the driver will quote it as `a.b.c`
+                col_name = ".".join([f"`{part}`" for part in parent.split(".")])
+                col.name = ColumnName(f"{col_name}.`{col.name.root}`")
+                # Set `_quote` to False to avoid quoting the column name again when compiled
+                sqa_col = build_orm_col(
+                    idx=1,
+                    col=col,
+                    table_service_type=DatabaseServiceType.Databricks,
+                    _quote=False,
+                )
+                sqa_col._set_parent(  # pylint: disable=protected-access
                     self.table.__table__
                 )
-                columns_list.append(col)
+                columns_list.append(sqa_col)
             else:
-                col = self._get_struct_columns(
-                    col.children, f"{parent}.{col.name.__root__}"
+                cols = self._get_struct_columns(
+                    col.children, f"{parent}.{col.name.root}"
                 )
-                columns_list.extend(col)
+                columns_list.extend(cols)
         return columns_list
 
     def get_columns(self) -> Column:
         """Get columns from table"""
         columns = []
-        for idx, column in enumerate(self.table_entity.columns):
-            if column.dataType == DataType.STRUCT:
+        for idx, column_obj in enumerate(self.table_entity.columns):
+            if column_obj.dataType == DataType.STRUCT:
                 columns.extend(
-                    self._get_struct_columns(column.children, column.name.__root__)
+                    self._get_struct_columns(column_obj.children, column_obj.name.root)
                 )
             else:
-                col = build_orm_col(idx, column, DatabaseServiceType.Databricks)
+                col = build_orm_col(idx, column_obj, DatabaseServiceType.Databricks)
                 col._set_parent(  # pylint: disable=protected-access
                     self.table.__table__
                 )

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -19,7 +19,7 @@ import concurrent.futures
 import threading
 import traceback
 from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Dict, List, Optional
 
 from sqlalchemy import Column, inspect, text
@@ -349,7 +349,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     crs.scalar()
                 )  # raise MultipleResultsFound if more than one row is returned
                 custom_metrics.append(
-                    CustomMetricProfile(name=metric.name.__root__, value=row)
+                    CustomMetricProfile(name=metric.name.root, value=row)
                 )
 
             except Exception as exc:
@@ -506,9 +506,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                         profile_results["columns"][column].update(
                             {
                                 "name": column,
-                                "timestamp": int(
-                                    datetime.now(tz=timezone.utc).timestamp() * 1000
-                                ),
+                                "timestamp": int(datetime.now().timestamp() * 1000),
                                 **profile,
                             }
                         )

metadata/profiler/metrics/static/mean.py

@@ -52,6 +52,15 @@ def _(element, compiler, **kw):
     return f"if(isNaN(avg({proc})), null, avg({proc}))"
 
 
+@compiles(avg, Dialects.Redshift)
+def _(element, compiler, **kw):
+    """
+    Cast to decimal to get around potential integer overflow error
+    """
+    proc = compiler.process(element.clauses, **kw)
+    return f"avg(CAST({proc} AS DECIMAL(38,0)))"
+
+
 @compiles(avg, Dialects.MSSQL)
 def _(element, compiler, **kw):
     """

metadata/profiler/metrics/system/queries/snowflake.py

@@ -101,7 +101,7 @@ def get_identifiers(
         )
         es_tables = search_table_entities(
             metadata=ometa_client,
-            service_name=db_service.fullyQualifiedName.__root__,
+            service_name=db_service.fullyQualifiedName.root,
             database=database_name,
             database_schema=schema_name,
             table=table_name,

metadata/profiler/metrics/window/percentille_mixin.py

@@ -1,4 +1,4 @@
-"""function calls shared accross all percentile metrics"""
+"""function calls shared across all percentile metrics"""
 
 from metadata.profiler.orm.functions.median import MedianFn
 

metadata/profiler/orm/converter/base.py

@@ -64,7 +64,9 @@ def check_if_should_quote_column_name(table_service_type) -> Optional[bool]:
     return None
 
 
-def build_orm_col(idx: int, col: Column, table_service_type) -> sqlalchemy.Column:
+def build_orm_col(
+    idx: int, col: Column, table_service_type, *, _quote=None
+) -> sqlalchemy.Column:
     """
     Cook the ORM column from our metadata instance
     information.
@@ -76,16 +78,22 @@ def build_orm_col(idx: int, col: Column, table_service_type) -> sqlalchemy.Colum
     As this is only used for INSERT/UPDATE/DELETE,
     there is no impact for our read-only purposes.
     """
+    if _quote is not None:
+        quote = _quote
+    else:
+        quote = check_if_should_quote_column_name(
+            table_service_type
+        ) or check_snowflake_case_sensitive(table_service_type, col.name.root)
+
     return sqlalchemy.Column(
-        name=str(col.name.__root__),
+        name=str(col.name.root),
         type_=converter_registry[table_service_type]().map_types(
             col, table_service_type
         ),
         primary_key=not bool(idx),  # The first col seen is used as PK
-        quote=check_if_should_quote_column_name(table_service_type)
-        or check_snowflake_case_sensitive(table_service_type, col.name.__root__),
+        quote=quote,
         key=str(
-            col.name.__root__
+            col.name.root
         ).lower(),  # Add lowercase column name as key for snowflake case sensitive columns
     )
 
@@ -107,16 +115,16 @@ def ometa_to_sqa_orm(
     )  # satisfy mypy
     cols = {
         (
-            col.name.__root__ + "_"
-            if col.name.__root__ in SQA_RESERVED_ATTRIBUTES
-            else col.name.__root__
+            col.name.root + "_"
+            if col.name.root in SQA_RESERVED_ATTRIBUTES
+            else col.name.root
         ): build_orm_col(idx, col, table.serviceType)
         for idx, col in enumerate(table.columns)
     }
 
     orm_database_name = get_orm_database(table, metadata)
     orm_schema_name = get_orm_schema(table, metadata)
-    orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.__root__}".replace(
+    orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
         ".", "_"
     )
 
@@ -125,7 +133,7 @@ def ometa_to_sqa_orm(
         orm_name,  # Output class name
         (Base,),  # SQLAlchemy declarative base
         {
-            "__tablename__": str(table.name.__root__),
+            "__tablename__": str(table.name.root),
             "__table_args__": {
                 # SQLite does not support schemas
                 "schema": orm_schema_name
@@ -133,7 +141,7 @@ def ometa_to_sqa_orm(
                 else None,
                 "extend_existing": True,  # Recreates the table ORM object if it already exists. Useful for testing
                 "quote": check_snowflake_case_sensitive(
-                    table.serviceType, table.name.__root__
+                    table.serviceType, table.name.root
                 ),
             },
             **cols,
@@ -166,7 +174,7 @@ def get_orm_schema(table: Table, metadata: OpenMetadata) -> str:
         entity=DatabaseSchema, entity_id=table.databaseSchema.id
     )
 
-    return str(schema.name.__root__)
+    return str(schema.name.root)
 
 
 def get_orm_database(table: Table, metadata: OpenMetadata) -> str:
@@ -184,4 +192,4 @@ def get_orm_database(table: Table, metadata: OpenMetadata) -> str:
         entity=Database, entity_id=table.database.id
     )
 
-    return str(database.name.__root__)
+    return str(database.name.root)

metadata/profiler/orm/converter/common.py

@@ -12,7 +12,6 @@
 """
 Common Class For Profiler Converter.
 """
-
 from typing import Dict, Set
 
 import sqlalchemy
@@ -51,8 +50,8 @@ class CommonMapTypes:
         DataType.CHAR: sqlalchemy.CHAR,
         DataType.VARCHAR: sqlalchemy.VARCHAR,
         DataType.BOOLEAN: sqlalchemy.BOOLEAN,
-        DataType.BINARY: sqlalchemy.LargeBinary,
-        DataType.VARBINARY: sqlalchemy.VARBINARY,
+        DataType.BINARY: CustomTypes.BYTES.value,
+        DataType.VARBINARY: CustomTypes.BYTES.value,
         DataType.ARRAY: CustomTypes.ARRAY.value,
         DataType.BLOB: CustomTypes.BYTES.value,
         DataType.LONGBLOB: sqlalchemy.LargeBinary,
@@ -81,7 +80,7 @@ class CommonMapTypes:
         return self.return_custom_type(col, table_service_type)
 
     def return_custom_type(self, col: Column, _):
-        return self._TYPE_MAP.get(col.dataType)
+        return self._TYPE_MAP.get(col.dataType, CustomTypes.UNDETERMINED.value)
 
     @staticmethod
     def map_sqa_to_om_types() -> Dict[TypeEngine, Set[DataType]]:

metadata/profiler/orm/functions/datetime.py

@@ -188,7 +188,7 @@ def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
             "Bigquery does not support `month` or `year` interval for table partitioned on timestamp",
             "field types. You can set the `interval_unit to day or hour directly from OpenMetadata UI`."
             # pylint: disable=line-too-long
-            "Visit https://docs.open-metadata.org/connectors/ingestion/workflows/profiler#4-updating-profiler-setting-at-the-table-level for more details.",
+            "Visit https://docs.open-metadata.org/how-to-guides/data-quality-observability/profiler/workflow#4-updating-profiler-setting-at-the-table-level for more details.",
         )
 
     return (

metadata/profiler/orm/functions/sum.py

@@ -32,6 +32,13 @@ def _(element, compiler, **kw):
     return f"SUM(CAST({proc} AS BIGINT))"
 
 
+@compiles(SumFn, Dialects.Redshift)
+def _(element, compiler, **kw):
+    """Cast to Decimal to address overflow error from summing 32-bit int in most database dialects"""
+    proc = compiler.process(element.clauses, **kw)
+    return f"SUM(CAST({proc} AS Decimal(38,0)))"
+
+
 @compiles(SumFn, Dialects.Athena)
 @compiles(SumFn, Dialects.Trino)
 @compiles(SumFn, Dialects.Presto)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -295,7 +295,7 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
 
         where_clause = [
             Column("project_id")
-            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            == self.conn_config.credentials.gcpConfig.projectId.root,
             Column("table_schema") == self.schema_name,
             Column("table_name") == self.table_name,
         ]
@@ -324,21 +324,20 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
         columns = [
             Column("row_count").label("rowCount"),
             Column("size_bytes").label("sizeInBytes"),
-            Column("creation_time").label("createDateTime"),
+            func.TIMESTAMP_MILLIS(Column("creation_time")).label(CREATE_DATETIME),
             *self._get_col_names_and_count(),
         ]
         where_clause = [
             Column("project_id")
-            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            == self.conn_config.credentials.gcpConfig.projectId.root,
             Column("dataset_id") == self.schema_name,
             Column("table_id") == self.table_name,
         ]
-
         query = self._build_query(
             columns,
             self._build_table(
                 "__TABLES__",
-                f"{self.conn_config.credentials.gcpConfig.projectId.__root__}.{self.schema_name}",
+                f"{self.conn_config.credentials.gcpConfig.projectId.root}.{self.schema_name}",
             ),
             where_clause,
         )

metadata/profiler/orm/registry.py

@@ -26,6 +26,7 @@ from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString
 from metadata.profiler.orm.types.custom_image import CustomImage
 from metadata.profiler.orm.types.custom_ip import CustomIP
 from metadata.profiler.orm.types.custom_timestamp import CustomTimestamp
+from metadata.profiler.orm.types.undetermined_type import UndeterminedType
 from metadata.profiler.orm.types.uuid import UUIDString
 from metadata.profiler.registry import TypeRegistry
 
@@ -39,6 +40,7 @@ class CustomTypes(TypeRegistry):
     IMAGE = CustomImage
     IP = CustomIP
     SQADATETIMERANGE = CustomDateTimeRange
+    UNDETERMINED = UndeterminedType
 
 
 class Dialects(Enum):
@@ -97,6 +99,8 @@ NOT_COMPUTE = {
     DataType.JSON.value,
     CustomTypes.ARRAY.value.__name__,
     CustomTypes.SQADATETIMERANGE.value.__name__,
+    DataType.XML.value,
+    CustomTypes.UNDETERMINED.value.__name__,
 }
 FLOAT_SET = {sqlalchemy.types.DECIMAL, sqlalchemy.types.FLOAT}
 
@@ -146,6 +150,7 @@ def is_date_time(_type) -> bool:
         issubclass(_type.__class__, Date)
         or issubclass(_type.__class__, Time)
         or issubclass(_type.__class__, DateTime)
+        or issubclass(_type.__class__, CustomTimestamp)
     )
 
 

metadata/profiler/orm/types/custom_hex_byte_string.py

@@ -42,9 +42,11 @@ class HexByteString(TypeDecorator):
         Make sure the data is of correct type
         """
         if not isinstance(value, (bytes, bytearray)):
-            raise TypeError("HexByteString columns support only bytes values.")
+            raise TypeError(
+                f"HexByteString columns support only bytes values. Received {type(value).__name__}."
+            )
 
-    def process_result_value(self, value: str, dialect) -> Optional[str]:
+    def process_result_value(self, value: Optional[bytes], dialect) -> Optional[str]:
         """This is executed during result retrieval
 
         Args:
@@ -64,8 +66,8 @@ class HexByteString(TypeDecorator):
                 value = bytes_value.decode(encoding=detected_encoding)
                 return value
             except Exception as exc:
+                logger.debug("Failed to parse bytes valud as string: %s", exc)
                 logger.debug(traceback.format_exc())
-                logger.error(exc)
 
         return value.hex()
 

metadata/profiler/orm/types/undetermined_type.py

@@ -0,0 +1,38 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# pylint: disable=abstract-method
+
+"""
+Undetermined types for cases where we dont have typ mappings
+"""
+from sqlalchemy.sql.sqltypes import String, TypeDecorator
+
+
+class UndeterminedType(TypeDecorator):
+    """A fallback type for undetermined types returned from the database"""
+
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return str
+
+    def process_result_value(self, value, _):
+        """
+        We have no idea what is this type. So we just casr
+        """
+        return (
+            f"OPENMETADATA_UNDETERMIND[{str(value)}]"
+            if value
+            else "OPENMETADATA_UNDETERMIND[]"
+        )

metadata/profiler/processor/core.py

@@ -40,6 +40,7 @@ from metadata.generated.schema.settings.settings import Settings
 from metadata.generated.schema.tests.customMetric import (
     CustomMetric as CustomMetricEntity,
 )
+from metadata.generated.schema.type.basic import Timestamp
 from metadata.profiler.api.models import ProfilerResponse, ThreadPoolMetrics
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
 from metadata.profiler.metrics.core import (
@@ -51,6 +52,7 @@ from metadata.profiler.metrics.core import (
     TMetric,
 )
 from metadata.profiler.metrics.static.row_count import RowCount
+from metadata.profiler.orm.functions.table_metric_computer import CREATE_DATETIME
 from metadata.profiler.orm.registry import NOT_COMPUTE
 from metadata.profiler.processor.metric_filter import MetricFilter
 from metadata.profiler.processor.sample_data_handler import upload_sample_data
@@ -105,7 +107,7 @@ class Profiler(Generic[TMetric]):
         self.include_columns = include_columns
         self.exclude_columns = exclude_columns
         self._metrics = metrics
-        self._profile_date = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
+        self._profile_ts = Timestamp(int(datetime.now().timestamp() * 1000))
         self.profile_sample_config = self.profiler_interface.profile_sample_config
 
         self.metric_filter = MetricFilter(
@@ -150,8 +152,8 @@ class Profiler(Generic[TMetric]):
         return self._get_included_columns()
 
     @property
-    def profile_date(self) -> datetime:
-        return self._profile_date
+    def profile_ts(self) -> Timestamp:
+        return self._profile_ts
 
     @property
     def columns(self) -> List[Column]:
@@ -218,7 +220,7 @@ class Profiler(Generic[TMetric]):
                     return
 
         raise RuntimeError(
-            f"No profile data computed for {self.profiler_interface.table_entity.fullyQualifiedName.__root__}"
+            f"No profile data computed for {self.profiler_interface.table_entity.fullyQualifiedName.root}"
         )
 
     def get_custom_metrics(
@@ -240,7 +242,7 @@ class Profiler(Generic[TMetric]):
             (
                 clmn
                 for clmn in self.profiler_interface.table_entity.columns
-                if clmn.name.__root__ == column_name
+                if clmn.name.root == column_name
             ),
             None,
         )
@@ -486,7 +488,7 @@ class Profiler(Generic[TMetric]):
 
         if self.source_config.computeMetrics:
             logger.debug(
-                f"Computing profile metrics for {self.profiler_interface.table_entity.fullyQualifiedName.__root__}..."
+                f"Computing profile metrics for {self.profiler_interface.table_entity.fullyQualifiedName.root}..."
             )
             self.compute_metrics()
 
@@ -517,7 +519,7 @@ class Profiler(Generic[TMetric]):
         try:
             logger.debug(
                 "Fetching sample data for "
-                f"{self.profiler_interface.table_entity.fullyQualifiedName.__root__}..."  # type: ignore
+                f"{self.profiler_interface.table_entity.fullyQualifiedName.root}..."  # type: ignore
             )
             table_data = self.profiler_interface.fetch_sample_data(
                 self.table, self.columns
@@ -568,22 +570,26 @@ class Profiler(Generic[TMetric]):
                     **self.column_results.get(
                         col.name
                         if not isinstance(col.name, ColumnName)
-                        else col.name.__root__
+                        else col.name.root
                     )
                 )
                 for col in self.columns
                 if self.column_results.get(
-                    col.name
-                    if not isinstance(col.name, ColumnName)
-                    else col.name.__root__
+                    col.name if not isinstance(col.name, ColumnName) else col.name.root
                 )
             ]
 
+            raw_create_date: Optional[datetime] = self._table_results.get(
+                CREATE_DATETIME
+            )
+            if raw_create_date:
+                raw_create_date = raw_create_date.replace(tzinfo=timezone.utc)
+
             table_profile = TableProfile(
-                timestamp=self.profile_date,
+                timestamp=self.profile_ts,
                 columnCount=self._table_results.get("columnCount"),
                 rowCount=self._table_results.get(RowCount.name()),
-                createDateTime=self._table_results.get("createDateTime"),
+                createDateTime=raw_create_date,
                 sizeInByte=self._table_results.get("sizeInBytes"),
                 profileSample=(
                     self.profile_sample_config.profile_sample

metadata/profiler/processor/processor.py

@@ -43,8 +43,8 @@ class ProfilerProcessor(Processor):
         super().__init__()
 
         self.config = config
-        self.profiler_config = ProfilerProcessorConfig.parse_obj(
-            self.config.processor.dict().get("config")
+        self.profiler_config = ProfilerProcessorConfig.model_validate(
+            self.config.processor.model_dump().get("config")
         )
         self.source_config: DatabaseServiceProfilerPipeline = cast(
             DatabaseServiceProfilerPipeline, self.config.source.sourceConfig.config
@@ -64,8 +64,8 @@ class ProfilerProcessor(Processor):
         except Exception as exc:
             self.status.failed(
                 StackTraceError(
-                    name=record.entity.fullyQualifiedName.__root__,
-                    error=f"Unexpected exception processing entity {record.entity.fullyQualifiedName.__root__}: {exc}",
+                    name=record.entity.fullyQualifiedName.root,
+                    error=f"Unexpected exception processing entity {record.entity.fullyQualifiedName.root}: {exc}",
                     stackTrace=traceback.format_exc(),
                 )
             )