openmetadata-ingestion 1.4.7.2__py3-none-any.whl → 1.5.0.0__py3-none-any.whl

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -16,11 +16,12 @@ supporting sqlalchemy abstraction layer
 """
 
 import concurrent.futures
+import math
 import threading
 import traceback
 from collections import defaultdict
-from datetime import datetime, timezone
-from typing import Dict, List, Optional
+from datetime import datetime
+from typing import Any, Dict, List, Optional
 
 from sqlalchemy import Column, inspect, text
 from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
@@ -349,7 +350,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     crs.scalar()
                 )  # raise MultipleResultsFound if more than one row is returned
                 custom_metrics.append(
-                    CustomMetricProfile(name=metric.name.__root__, value=row)
+                    CustomMetricProfile(name=metric.name.root, value=row)
                 )
 
             except Exception as exc:
@@ -450,6 +451,16 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     column=metric_func.column,
                     sample=sample,
                 )
+                if row and isinstance(row, dict):
+                    row = self._validate_nulls(row)
+
+                # System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
+                if row and metric_func.metric_type == MetricTypes.System:
+                    row = [
+                        self._validate_nulls(r) if isinstance(r, dict) else r
+                        for r in row
+                    ]
+
             except Exception as exc:
                 error = (
                     f"{metric_func.column if metric_func.column is not None else metric_func.table.__tablename__} "
@@ -467,6 +478,17 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
 
             return row, column, metric_func.metric_type.value
 
+    @staticmethod
+    def _validate_nulls(row: Dict[str, Any]) -> Dict[str, Any]:
+        """Detect if we are computing NaNs and replace them with None"""
+        for k, v in row.items():
+            if isinstance(v, float) and math.isnan(v):
+                logger.warning(
+                    "NaN data detected and will be cast to null in OpenMetadata to maintain database parity"
+                )
+                row[k] = None
+        return row
+
     # pylint: disable=use-dict-literal
     def get_all_metrics(
         self,
@@ -506,9 +528,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                         profile_results["columns"][column].update(
                             {
                                 "name": column,
-                                "timestamp": int(
-                                    datetime.now(tz=timezone.utc).timestamp() * 1000
-                                ),
+                                "timestamp": int(datetime.now().timestamp() * 1000),
                                 **profile,
                             }
                         )

metadata/profiler/metrics/composed/null_ratio.py

@@ -49,12 +49,9 @@ class NullRatio(ComposedMetric):
         Safely compute null ratio based on the profiler
         results of other Metrics
         """
-        import pandas as pd
 
-        res_count = res.get(Count.name())
-        res_null = res.get(NullCount.name())
-
-        if not pd.isnull(res_count) and not pd.isnull(res_null):
-            result = res_null / (res_null + res_count)
-            return None if pd.isnull(result) else result
-        return None
+        count = res.get(Count.name())
+        null_count = res.get(NullCount.name())
+        if count + null_count == 0:
+            return None
+        return null_count / (null_count + count)

metadata/profiler/metrics/hybrid/histogram.py

@@ -25,7 +25,11 @@ from metadata.profiler.metrics.static.count import Count
 from metadata.profiler.metrics.static.max import Max
 from metadata.profiler.metrics.static.min import Min
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
+from metadata.profiler.orm.registry import (
+    is_concatenable,
+    is_quantifiable,
+    is_value_non_numeric,
+)
 from metadata.utils.helpers import format_large_string_numbers
 from metadata.utils.logger import profiler_logger
 
@@ -120,7 +124,6 @@ class Histogram(HybridMetric):
             # freedman-diaconis rule
             bin_width = self._get_bin_width(float(res_iqr), res_row_count)  # type: ignore
             num_bins = math.ceil((res_max - res_min) / bin_width)  # type: ignore
-
         # sturge's rule
         if res_iqr is None or num_bins > max_bin_count:
             num_bins = int(math.ceil(math.log2(res_row_count) + 1))
@@ -148,7 +151,10 @@ class Histogram(HybridMetric):
                 "We are missing the session attribute to compute the Histogram."
             )
 
-        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)):
+        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)) or (
+            is_value_non_numeric(res.get(Min.name()))
+            or is_value_non_numeric(res.get(Max.name()))
+        ):
             return None
 
         # get the metric need for the freedman-diaconis rule

metadata/profiler/metrics/static/mean.py

@@ -15,7 +15,7 @@ AVG Metric definition
 from functools import partial
 from typing import Callable, List, Optional, cast
 
-from sqlalchemy import column, func
+from sqlalchemy import column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
@@ -39,20 +39,28 @@ from metadata.utils.logger import profiler_logger
 logger = profiler_logger()
 
 
-# pylint: disable=invalid-name
-class avg(GenericFunction):
+class AvgFn(GenericFunction):
     name = "avg"
     inherit_cache = CACHE
 
 
-@compiles(avg, Dialects.ClickHouse)
+@compiles(AvgFn, Dialects.ClickHouse)
 def _(element, compiler, **kw):
     """Handle case for empty table. If empty, clickhouse returns NaN"""
     proc = compiler.process(element.clauses, **kw)
     return f"if(isNaN(avg({proc})), null, avg({proc}))"
 
 
-@compiles(avg, Dialects.MSSQL)
+@compiles(AvgFn, Dialects.Redshift)
+def _(element, compiler, **kw):
+    """
+    Cast to decimal to get around potential integer overflow error
+    """
+    proc = compiler.process(element.clauses, **kw)
+    return f"avg(CAST({proc} AS DECIMAL(38,0)))"
+
+
+@compiles(AvgFn, Dialects.MSSQL)
 def _(element, compiler, **kw):
     """
     Cast to decimal to get around potential integer overflow error -
@@ -62,7 +70,7 @@ def _(element, compiler, **kw):
     return f"avg(cast({proc} as decimal))"
 
 
-@compiles(avg, Dialects.Trino)
+@compiles(AvgFn, Dialects.Trino)
 def _(element, compiler, **kw):
     proc = compiler.process(element.clauses, **kw)
     first_clause = element.clauses.clauses[0]
@@ -99,10 +107,10 @@ class Mean(StaticMetric):
     def fn(self):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
-            return func.avg(column(self.col.name, self.col.type))
+            return AvgFn(column(self.col.name, self.col.type))
 
         if is_concatenable(self.col.type):
-            return func.avg(LenFn(column(self.col.name, self.col.type)))
+            return AvgFn(LenFn(column(self.col.name, self.col.type)))
 
         logger.debug(
             f"Don't know how to process type {self.col.type} when computing MEAN"

metadata/profiler/metrics/static/null_count.py

@@ -16,6 +16,7 @@ Null Count Metric definition
 
 
 from sqlalchemy import case, column
+from sqlalchemy.sql.functions import coalesce
 
 from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
@@ -48,8 +49,9 @@ class NullCount(StaticMetric):
     @_label
     def fn(self):
         """sqlalchemy function"""
-        return SumFn(
-            case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)
+        return coalesce(
+            SumFn(case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)),
+            0,
         )
 
     def df_fn(self, dfs=None):

metadata/profiler/metrics/system/queries/snowflake.py

@@ -101,7 +101,7 @@ def get_identifiers(
         )
         es_tables = search_table_entities(
             metadata=ometa_client,
-            service_name=db_service.fullyQualifiedName.__root__,
+            service_name=db_service.fullyQualifiedName.root,
             database=database_name,
             database_schema=schema_name,
             table=table_name,

metadata/profiler/metrics/window/percentille_mixin.py

@@ -1,4 +1,4 @@
-"""function calls shared accross all percentile metrics"""
+"""function calls shared across all percentile metrics"""
 
 from metadata.profiler.orm.functions.median import MedianFn
 

metadata/profiler/orm/converter/base.py

@@ -64,7 +64,9 @@ def check_if_should_quote_column_name(table_service_type) -> Optional[bool]:
     return None
 
 
-def build_orm_col(idx: int, col: Column, table_service_type) -> sqlalchemy.Column:
+def build_orm_col(
+    idx: int, col: Column, table_service_type, *, _quote=None
+) -> sqlalchemy.Column:
     """
     Cook the ORM column from our metadata instance
     information.
@@ -76,16 +78,22 @@ def build_orm_col(idx: int, col: Column, table_service_type) -> sqlalchemy.Colum
     As this is only used for INSERT/UPDATE/DELETE,
     there is no impact for our read-only purposes.
     """
+    if _quote is not None:
+        quote = _quote
+    else:
+        quote = check_if_should_quote_column_name(
+            table_service_type
+        ) or check_snowflake_case_sensitive(table_service_type, col.name.root)
+
     return sqlalchemy.Column(
-        name=str(col.name.__root__),
+        name=str(col.name.root),
         type_=converter_registry[table_service_type]().map_types(
             col, table_service_type
         ),
         primary_key=not bool(idx),  # The first col seen is used as PK
-        quote=check_if_should_quote_column_name(table_service_type)
-        or check_snowflake_case_sensitive(table_service_type, col.name.__root__),
+        quote=quote,
         key=str(
-            col.name.__root__
+            col.name.root
         ).lower(),  # Add lowercase column name as key for snowflake case sensitive columns
     )
 
@@ -107,16 +115,16 @@ def ometa_to_sqa_orm(
     )  # satisfy mypy
     cols = {
         (
-            col.name.__root__ + "_"
-            if col.name.__root__ in SQA_RESERVED_ATTRIBUTES
-            else col.name.__root__
+            col.name.root + "_"
+            if col.name.root in SQA_RESERVED_ATTRIBUTES
+            else col.name.root
         ): build_orm_col(idx, col, table.serviceType)
         for idx, col in enumerate(table.columns)
     }
 
     orm_database_name = get_orm_database(table, metadata)
     orm_schema_name = get_orm_schema(table, metadata)
-    orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.__root__}".replace(
+    orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
         ".", "_"
     )
 
@@ -125,7 +133,7 @@ def ometa_to_sqa_orm(
         orm_name,  # Output class name
         (Base,),  # SQLAlchemy declarative base
         {
-            "__tablename__": str(table.name.__root__),
+            "__tablename__": str(table.name.root),
             "__table_args__": {
                 # SQLite does not support schemas
                 "schema": orm_schema_name
@@ -133,7 +141,7 @@ def ometa_to_sqa_orm(
                 else None,
                 "extend_existing": True,  # Recreates the table ORM object if it already exists. Useful for testing
                 "quote": check_snowflake_case_sensitive(
-                    table.serviceType, table.name.__root__
+                    table.serviceType, table.name.root
                 ),
             },
             **cols,
@@ -166,7 +174,7 @@ def get_orm_schema(table: Table, metadata: OpenMetadata) -> str:
         entity=DatabaseSchema, entity_id=table.databaseSchema.id
     )
 
-    return str(schema.name.__root__)
+    return str(schema.name.root)
 
 
 def get_orm_database(table: Table, metadata: OpenMetadata) -> str:
@@ -184,4 +192,4 @@ def get_orm_database(table: Table, metadata: OpenMetadata) -> str:
         entity=Database, entity_id=table.database.id
     )
 
-    return str(database.name.__root__)
+    return str(database.name.root)

metadata/profiler/orm/converter/common.py

@@ -12,7 +12,6 @@
 """
 Common Class For Profiler Converter.
 """
-
 from typing import Dict, Set
 
 import sqlalchemy
@@ -51,8 +50,8 @@ class CommonMapTypes:
         DataType.CHAR: sqlalchemy.CHAR,
         DataType.VARCHAR: sqlalchemy.VARCHAR,
         DataType.BOOLEAN: sqlalchemy.BOOLEAN,
-        DataType.BINARY: sqlalchemy.LargeBinary,
-        DataType.VARBINARY: sqlalchemy.VARBINARY,
+        DataType.BINARY: CustomTypes.BYTES.value,
+        DataType.VARBINARY: CustomTypes.BYTES.value,
         DataType.ARRAY: CustomTypes.ARRAY.value,
         DataType.BLOB: CustomTypes.BYTES.value,
         DataType.LONGBLOB: sqlalchemy.LargeBinary,
@@ -81,7 +80,7 @@ class CommonMapTypes:
         return self.return_custom_type(col, table_service_type)
 
     def return_custom_type(self, col: Column, _):
-        return self._TYPE_MAP.get(col.dataType)
+        return self._TYPE_MAP.get(col.dataType, CustomTypes.UNDETERMINED.value)
 
     @staticmethod
     def map_sqa_to_om_types() -> Dict[TypeEngine, Set[DataType]]:

metadata/profiler/orm/functions/datetime.py

@@ -188,7 +188,7 @@ def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
             "Bigquery does not support `month` or `year` interval for table partitioned on timestamp",
             "field types. You can set the `interval_unit to day or hour directly from OpenMetadata UI`."
             # pylint: disable=line-too-long
-            "Visit https://docs.open-metadata.org/connectors/ingestion/workflows/profiler#4-updating-profiler-setting-at-the-table-level for more details.",
+            "Visit https://docs.open-metadata.org/how-to-guides/data-quality-observability/profiler/workflow#4-updating-profiler-setting-at-the-table-level for more details.",
         )
 
     return (

metadata/profiler/orm/functions/median.py

@@ -28,12 +28,19 @@ logger = profiler_logger()
 class MedianFn(FunctionElement):
     inherit_cache = CACHE
 
+    @staticmethod
+    def default_fn(elements, compiler, **kwargs):  # pylint: disable=unused-argument
+        col = compiler.process(elements.clauses.clauses[0])
+        percentile = elements.clauses.clauses[2].value
+        return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (
+            percentile,
+            col,
+        )
+
 
 @compiles(MedianFn)
-def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
-    col = compiler.process(elements.clauses.clauses[0])
-    percentile = elements.clauses.clauses[2].value
-    return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (percentile, col)
+def _(elements, compiler, **kwargs):
+    return MedianFn.default_fn(elements, compiler, **kwargs)
 
 
 @compiles(MedianFn, Dialects.BigQuery)

metadata/profiler/orm/functions/sum.py

@@ -32,13 +32,20 @@ def _(element, compiler, **kw):
     return f"SUM(CAST({proc} AS BIGINT))"
 
 
+@compiles(SumFn, Dialects.Redshift)
+def _(element, compiler, **kw):
+    """Cast to Decimal to address overflow error from summing 32-bit int in most database dialects"""
+    proc = compiler.process(element.clauses, **kw)
+    return f"SUM(CAST({proc} AS Decimal(38,0)))"
+
+
 @compiles(SumFn, Dialects.Athena)
 @compiles(SumFn, Dialects.Trino)
 @compiles(SumFn, Dialects.Presto)
 def _(element, compiler, **kw):
     """Cast to DECIMAL to address cannot cast nan to bigint"""
     proc = compiler.process(element.clauses, **kw)
-    return f"SUM(TRY_CAST({proc} AS DECIMAL))"
+    return f"COALESCE(SUM(CAST({proc} AS DECIMAL)),0)"
 
 
 @compiles(SumFn, Dialects.BigQuery)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -194,7 +194,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("object_name").label("table_name"),
                     Column("created"),
                 ],
-                self._build_table("all_objects", None),
+                self._build_table("DBA_OBJECTS", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("object_name")) == self.table_name.lower(),
@@ -209,7 +209,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("table_name"),
                     Column("NUM_ROWS"),
                 ],
-                self._build_table("all_tables", None),
+                self._build_table("DBA_TABLES", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("table_name")) == self.table_name.lower(),
@@ -295,7 +295,7 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
 
         where_clause = [
             Column("project_id")
-            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            == self.conn_config.credentials.gcpConfig.projectId.root,
             Column("table_schema") == self.schema_name,
             Column("table_name") == self.table_name,
         ]
@@ -324,21 +324,20 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
         columns = [
             Column("row_count").label("rowCount"),
             Column("size_bytes").label("sizeInBytes"),
-            Column("creation_time").label("createDateTime"),
+            func.TIMESTAMP_MILLIS(Column("creation_time")).label(CREATE_DATETIME),
             *self._get_col_names_and_count(),
         ]
         where_clause = [
             Column("project_id")
-            == self.conn_config.credentials.gcpConfig.projectId.__root__,
+            == self.conn_config.credentials.gcpConfig.projectId.root,
             Column("dataset_id") == self.schema_name,
             Column("table_id") == self.table_name,
         ]
-
         query = self._build_query(
             columns,
             self._build_table(
                 "__TABLES__",
-                f"{self.conn_config.credentials.gcpConfig.projectId.__root__}.{self.schema_name}",
+                f"{self.conn_config.credentials.gcpConfig.projectId.root}.{self.schema_name}",
             ),
             where_clause,
         )

metadata/profiler/orm/registry.py

@@ -13,6 +13,8 @@
 Custom types' registry for easy access
 without having an import mess
 """
+import math
+
 import sqlalchemy
 from sqlalchemy import Date, DateTime, Integer, Numeric, Time
 from sqlalchemy.sql.sqltypes import Concatenable, Enum
@@ -26,6 +28,7 @@ from metadata.profiler.orm.types.custom_hex_byte_string import HexByteString
 from metadata.profiler.orm.types.custom_image import CustomImage
 from metadata.profiler.orm.types.custom_ip import CustomIP
 from metadata.profiler.orm.types.custom_timestamp import CustomTimestamp
+from metadata.profiler.orm.types.undetermined_type import UndeterminedType
 from metadata.profiler.orm.types.uuid import UUIDString
 from metadata.profiler.registry import TypeRegistry
 
@@ -39,6 +42,7 @@ class CustomTypes(TypeRegistry):
     IMAGE = CustomImage
     IP = CustomIP
     SQADATETIMERANGE = CustomDateTimeRange
+    UNDETERMINED = UndeterminedType
 
 
 class Dialects(Enum):
@@ -97,6 +101,8 @@ NOT_COMPUTE = {
     DataType.JSON.value,
     CustomTypes.ARRAY.value.__name__,
     CustomTypes.SQADATETIMERANGE.value.__name__,
+    DataType.XML.value,
+    CustomTypes.UNDETERMINED.value.__name__,
 }
 FLOAT_SET = {sqlalchemy.types.DECIMAL, sqlalchemy.types.FLOAT}
 
@@ -146,6 +152,7 @@ def is_date_time(_type) -> bool:
         issubclass(_type.__class__, Date)
         or issubclass(_type.__class__, Time)
         or issubclass(_type.__class__, DateTime)
+        or issubclass(_type.__class__, CustomTimestamp)
     )
 
 
@@ -155,7 +162,9 @@ def is_quantifiable(_type) -> bool:
     """
     if isinstance(_type, DataType):
         return _type.value in QUANTIFIABLE_SET
-    return is_numeric(_type) or is_integer(_type)
+    return (
+        is_numeric(_type) or is_integer(_type) or getattr(_type, "quantifiable", False)
+    )
 
 
 def is_concatenable(_type) -> bool:
@@ -166,3 +175,12 @@ def is_concatenable(_type) -> bool:
     if isinstance(_type, DataType):
         return _type.value in CONCATENABLE_SET
     return issubclass(_type.__class__, Concatenable)
+
+
+def is_value_non_numeric(value) -> bool:
+    try:
+        if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
+            return True
+        return False
+    except Exception:
+        return False

metadata/profiler/orm/types/custom_hex_byte_string.py

@@ -42,9 +42,11 @@ class HexByteString(TypeDecorator):
         Make sure the data is of correct type
         """
         if not isinstance(value, (bytes, bytearray)):
-            raise TypeError("HexByteString columns support only bytes values.")
+            raise TypeError(
+                f"HexByteString columns support only bytes values. Received {type(value).__name__}."
+            )
 
-    def process_result_value(self, value: str, dialect) -> Optional[str]:
+    def process_result_value(self, value: Optional[bytes], dialect) -> Optional[str]:
         """This is executed during result retrieval
 
         Args:
@@ -64,8 +66,8 @@ class HexByteString(TypeDecorator):
                 value = bytes_value.decode(encoding=detected_encoding)
                 return value
             except Exception as exc:
+                logger.debug("Failed to parse bytes valud as string: %s", exc)
                 logger.debug(traceback.format_exc())
-                logger.error(exc)
 
         return value.hex()
 

metadata/profiler/orm/types/undetermined_type.py

@@ -0,0 +1,38 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# pylint: disable=abstract-method
+
+"""
+Undetermined types for cases where we dont have typ mappings
+"""
+from sqlalchemy.sql.sqltypes import String, TypeDecorator
+
+
+class UndeterminedType(TypeDecorator):
+    """A fallback type for undetermined types returned from the database"""
+
+    impl = String
+    cache_ok = True
+
+    @property
+    def python_type(self):
+        return str
+
+    def process_result_value(self, value, _):
+        """
+        We have no idea what is this type. So we just casr
+        """
+        return (
+            f"OPENMETADATA_UNDETERMIND[{str(value)}]"
+            if value
+            else "OPENMETADATA_UNDETERMIND[]"
+        )