openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0__py3-none-any.whl

metadata/profiler/metrics/static/mean.py

@@ -12,16 +12,17 @@
 """
 AVG Metric definition
 """
-# pylint: disable=duplicate-code
-
-
-from typing import List, cast
+from functools import partial
+from typing import Callable, List, Optional, cast
 
 from sqlalchemy import column, func
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
-from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
     FLOAT_SET,
@@ -32,6 +33,9 @@ from metadata.profiler.orm.registry import (
 )
 from metadata.utils.logger import profiler_logger
 
+# pylint: disable=duplicate-code
+
+
 logger = profiler_logger()
 
 
@@ -85,7 +89,7 @@ class Mean(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "mean"
+        return MetricType.mean.value
 
     @property
     def metric_type(self):
@@ -115,25 +119,23 @@ class Mean(StaticMetric):
 
         means = []
         weights = []
-
-        if is_quantifiable(self.col.type):
-            for df in dfs:
-                mean = df[self.col.name].mean()
-                if not pd.isnull(mean):
-                    means.append(mean)
-                    weights.append(df[self.col.name].count())
-
-        if is_concatenable(self.col.type):
-            length_vectorize_func = vectorize(len)
-            for df in dfs:
+        length_vectorize_func = vectorize(len)
+        for df in dfs:
+            processed_df = df[self.col.name].dropna()
+            try:
                 mean = None
-                if any(df[self.col.name]):
-                    mean = length_vectorize_func(
-                        df[self.col.name].dropna().astype(str)
-                    ).mean()
+                if is_quantifiable(self.col.type):
+                    mean = processed_df.mean()
+                if is_concatenable(self.col.type):
+                    mean = length_vectorize_func(processed_df.astype(str)).mean()
                 if not pd.isnull(mean):
                     means.append(mean)
-                    weights.append(df[self.col.name].dropna().count())
+                    weights.append(processed_df.count())
+            except Exception as err:
+                logger.debug(
+                    f"Error while computing mean for column {self.col.name}: {err}"
+                )
+                return None
 
         if means:
             return average(means, weights=weights)
@@ -142,3 +144,9 @@ class Mean(StaticMetric):
             f"Don't know how to process type {self.col.type} when computing MEAN"
         )
         return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.mean, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/min.py

@@ -12,13 +12,17 @@
 """
 Min Metric definition
 """
-# pylint: disable=duplicate-code
+from functools import partial
+from typing import Callable, Optional
 
 from sqlalchemy import TIME, column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
-from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import DataType, Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
     FLOAT_SET,
@@ -28,6 +32,8 @@ from metadata.profiler.orm.registry import (
     is_quantifiable,
 )
 
+# pylint: disable=duplicate-code
+
 
 class MinFn(GenericFunction):
     name = __qualname__
@@ -76,7 +82,7 @@ class Min(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "min"
+        return MetricType.min.value
 
     @_label
     def fn(self):
@@ -90,9 +96,22 @@ class Min(StaticMetric):
 
     def df_fn(self, dfs=None):
         """pandas function"""
+        import pandas as pd
+
         if is_quantifiable(self.col.type):
             return min((df[self.col.name].min() for df in dfs))
         if is_date_time(self.col.type):
-            min_ = min((df[self.col.name].min() for df in dfs))
-            return int(min_.timestamp() * 1000)
-        return 0
+            min_ = None
+            if self.col.type in {DataType.DATETIME, DataType.DATE}:
+                min_ = min((pd.to_datetime(df[self.col.name]).min() for df in dfs))
+                return None if pd.isnull(min_) else int(min_.timestamp() * 1000)
+            elif self.col.type == DataType.TIME:
+                min_ = min((pd.to_timedelta(df[self.col.name]).min() for df in dfs))
+                return None if pd.isnull(min_) else min_.seconds
+        return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.min, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/min_length.py

@@ -17,6 +17,7 @@ MIN_LENGTH Metric definition
 
 from sqlalchemy import column, func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class MinLength(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "minLength"
+        return MetricType.minLength.value
 
     @property
     def metric_type(self):
@@ -67,7 +68,7 @@ class MinLength(StaticMetric):
             min_length_list = []
 
             for df in dfs:
-                if any(df[self.col.name]):
+                if any(df[self.col.name].dropna()):
                     min_length_list.append(
                         length_vectorize_func(
                             df[self.col.name].dropna().astype(str)

metadata/profiler/metrics/static/not_like_count.py

@@ -16,6 +16,7 @@ Like Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -35,7 +36,7 @@ class NotLikeCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "notLikeCount"
+        return MetricType.notLikeCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/not_regexp_match_count.py

@@ -16,6 +16,7 @@ Regex Count Metric definition
 
 from sqlalchemy import case, column, not_
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class NotRegexCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "notRegexCount"
+        return MetricType.notRegexCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/null_count.py

@@ -17,6 +17,7 @@ Null Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -38,7 +39,7 @@ class NullCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "nullCount"
+        return MetricType.nullCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/null_missing_count.py

@@ -0,0 +1,73 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Null Count Metric definition
+"""
+# pylint: disable=duplicate-code
+
+
+from sqlalchemy import case, column
+
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.orm.functions.sum import SumFn
+
+
+class NullMissingCount(StaticMetric):
+    """
+    NULL + Empty COUNT Metric
+
+    Given a column, return the null count.
+
+    We are building a CASE WHEN structure:
+    ```
+    SUM(
+        CASE is not null THEN 1
+        ELSE 0
+    )
+    ```
+    """
+
+    @classmethod
+    def name(cls):
+        """
+        Returns the name of the metric.
+        """
+        return MetricType.nullCount.value
+
+    @property
+    def metric_type(self):
+        """
+        Returns the type of the metric.
+        """
+        return int
+
+    @_label
+    def fn(self):
+        """
+        Returns the SQLAlchemy function for calculating the metric.
+        """
+        return SumFn(
+            case(
+                [
+                    (column(self.col.name, self.col.type).is_(None), 1),
+                    (column(self.col.name, self.col.type).__eq__(""), 1),
+                ],
+                else_=0,
+            )
+        )
+
+    def df_fn(self, dfs=None):
+        """
+        Returns the pandas function for calculating the metric.
+        """
+        return sum(df[self.col.name].isnull().sum() for df in dfs)

metadata/profiler/metrics/static/regexp_match_count.py

@@ -16,6 +16,7 @@ Regex Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class RegexCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "regexCount"
+        return MetricType.regexCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/row_count.py

@@ -12,11 +12,13 @@
 """
 Table Count Metric definition
 """
-# pylint: disable=duplicate-code
-
+from typing import Callable
 
 from sqlalchemy import func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
 from metadata.profiler.metrics.core import StaticMetric, _label
 
 
@@ -29,7 +31,7 @@ class RowCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "rowCount"
+        return MetricType.rowCount.value
 
     @classmethod
     def is_col_metric(cls) -> bool:
@@ -50,3 +52,7 @@ class RowCount(StaticMetric):
     def df_fn(self, dfs=None):
         """pandas function"""
         return sum(len(df.index) for df in dfs)
+
+    @classmethod
+    def nosql_fn(cls, client: NoSQLAdaptor) -> Callable[[Table], int]:
+        return client.item_count

metadata/profiler/metrics/static/stddev.py

@@ -21,6 +21,7 @@ from sqlalchemy import column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import FunctionElement
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
@@ -103,7 +104,7 @@ class StdDev(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "stddev"
+        return MetricType.stddev.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/sum.py

@@ -12,15 +12,21 @@
 """
 SUM Metric definition
 """
-# pylint: disable=duplicate-code
+from functools import partial
+from typing import Callable, Optional
 
 from sqlalchemy import column
 
-from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
 
+# pylint: disable=duplicate-code
+
 
 class Sum(StaticMetric):
     """
@@ -33,7 +39,7 @@ class Sum(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "sum"
+        return MetricType.sum.value
 
     @_label
     def fn(self):
@@ -50,5 +56,17 @@ class Sum(StaticMetric):
         """pandas function"""
 
         if is_quantifiable(self.col.type):
-            return sum(df[self.col.name].sum() for df in dfs)
+            try:
+                return sum(df[self.col.name].sum() for df in dfs)
+            except (TypeError, ValueError):
+                try:
+                    return sum(df[self.col.name].astype(float).sum() for df in dfs)
+                except Exception:
+                    return None
         return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.sum, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/unique_count.py

@@ -12,11 +12,13 @@
 """
 Unique Count Metric definition
 """
+import json
 from typing import Optional
 
 from sqlalchemy import column, func
 from sqlalchemy.orm import DeclarativeMeta, Session
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import QueryMetric
 from metadata.profiler.orm.functions.unique_count import _unique_count_query_mapper
 from metadata.profiler.orm.registry import NOT_COMPUTE
@@ -34,7 +36,7 @@ class UniqueCount(QueryMetric):
 
     @classmethod
     def name(cls):
-        return "uniqueCount"
+        return MetricType.uniqueCount.value
 
     @property
     def metric_type(self):
@@ -59,8 +61,8 @@ class UniqueCount(QueryMetric):
         unique_count_query = _unique_count_query_mapper[session.bind.dialect.name](
             col, session, sample
         )
-        only_once_cte = unique_count_query.cte("only_once")
-        return session.query(func.count().label(self.name())).select_from(only_once_cte)
+        only_once_sub = unique_count_query.subquery("only_once")
+        return session.query(func.count().label(self.name())).select_from(only_once_sub)
 
     def df_fn(self, dfs=None):
         """
@@ -72,7 +74,14 @@ class UniqueCount(QueryMetric):
             counter = Counter()
             for df in dfs:
                 df_col_value = df[self.col.name].dropna().to_list()
-                counter.update(df_col_value)
+                try:
+                    counter.update(df_col_value)
+                except TypeError as err:
+                    if isinstance(df_col_value, list):
+                        for value in df_col_value:
+                            counter.update([json.dumps(value)])
+                    else:
+                        raise err
             return len([key for key, value in counter.items() if value == 1])
         except Exception as err:
             logger.debug(

metadata/profiler/metrics/system/system.py

@@ -20,6 +20,7 @@ from typing import Dict, List, Optional
 from sqlalchemy import text
 from sqlalchemy.orm import DeclarativeMeta, Session
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
@@ -430,7 +431,7 @@ class System(SystemMetric):
 
     @classmethod
     def name(cls):
-        return "system"
+        return MetricType.system.value
 
     def _manage_cache(self, max_size_in_bytes: int = MAX_SIZE_IN_BYTES) -> None:
         """manage cache and clears it if it exceeds the max size

metadata/profiler/metrics/window/first_quartile.py

@@ -18,6 +18,7 @@ from typing import List, cast
 
 from sqlalchemy import column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
@@ -38,7 +39,7 @@ class FirstQuartile(StaticMetric, PercentilMixin):
 
     @classmethod
     def name(cls):
-        return "firstQuartile"
+        return MetricType.firstQuartile.value
 
     @classmethod
     def is_window_metric(cls):

metadata/profiler/metrics/window/median.py

@@ -18,6 +18,7 @@ from typing import List, cast
 
 from sqlalchemy import column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
@@ -38,7 +39,7 @@ class Median(StaticMetric, PercentilMixin):
 
     @classmethod
     def name(cls):
-        return "median"
+        return MetricType.median.value
 
     @classmethod
     def is_window_metric(cls):
@@ -89,7 +90,12 @@ class Median(StaticMetric, PercentilMixin):
                     f"We recommend using a smaller sample size or partitionning."
                 )
                 return None
-            median = df.median()
+            try:
+                median = df.median()
+            except Exception as err:
+                logger.error(
+                    f"Unable to compute Median for {self.col.name} due to error: {err}"
+                )
             return None if pd.isnull(median) else median
         logger.debug(
             f"Don't know how to process type {self.col.type} when computing Median"

metadata/profiler/metrics/window/third_quartile.py

@@ -18,6 +18,7 @@ from typing import List, cast
 
 from sqlalchemy import column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
@@ -38,7 +39,7 @@ class ThirdQuartile(StaticMetric, PercentilMixin):
 
     @classmethod
     def name(cls):
-        return "thirdQuartile"
+        return MetricType.thirdQuartile.value
 
     @classmethod
     def is_window_metric(cls):

metadata/profiler/orm/converter/base.py

@@ -127,7 +127,10 @@ def ometa_to_sqa_orm(
         {
             "__tablename__": str(table.name.__root__),
             "__table_args__": {
-                "schema": orm_schema_name,
+                # SQLite does not support schemas
+                "schema": orm_schema_name
+                if table.serviceType != databaseService.DatabaseServiceType.SQLite
+                else None,
                 "extend_existing": True,  # Recreates the table ORM object if it already exists. Useful for testing
                 "quote": check_snowflake_case_sensitive(
                     table.serviceType, table.name.__root__

metadata/profiler/orm/converter/bigquery/converter.py

@@ -29,3 +29,14 @@ class BigqueryMapTypes(CommonMapTypes):
         ):
             return bigquery_type_mapper(self._TYPE_MAP, col)
         return super().return_custom_type(col, table_service_type)
+
+    @staticmethod
+    def map_sqa_to_om_types() -> dict:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+        from sqlalchemy_bigquery import STRUCT
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            STRUCT: DataType.STRUCT,
+        }

metadata/profiler/orm/converter/common.py

@@ -79,3 +79,50 @@ class CommonMapTypes:
 
     def return_custom_type(self, col: Column, _):
         return self._TYPE_MAP.get(col.dataType)
+
+    @staticmethod
+    def map_sqa_to_om_types() -> dict:
+        """returns an ORM type"""
+        return {
+            sqlalchemy.NUMERIC: {DataType.NUMBER, DataType.NUMERIC},
+            sqlalchemy.SMALLINT: {
+                DataType.TINYINT,
+                DataType.SMALLINT,
+                DataType.BYTEINT,
+            },
+            sqlalchemy.INT: {DataType.INT},
+            sqlalchemy.BIGINT: {DataType.BIGINT},
+            CustomTypes.BYTES.value: {DataType.BLOB, DataType.BYTES},
+            sqlalchemy.FLOAT: {DataType.FLOAT},
+            sqlalchemy.DECIMAL: {DataType.DOUBLE, DataType.DECIMAL},
+            CustomTypes.TIMESTAMP.value: {DataType.TIMESTAMP},
+            sqlalchemy.TIME: {DataType.TIME},
+            sqlalchemy.DATE: {DataType.DATE},
+            sqlalchemy.DATETIME: {DataType.DATETIME},
+            sqlalchemy.Interval: {DataType.INTERVAL},
+            sqlalchemy.String: {DataType.STRING},
+            sqlalchemy.TEXT: {DataType.TEXT, DataType.MEDIUMTEXT},
+            sqlalchemy.CHAR: {DataType.CHAR},
+            sqlalchemy.VARCHAR: {DataType.VARCHAR},
+            sqlalchemy.BOOLEAN: {DataType.BOOLEAN},
+            sqlalchemy.LargeBinary: {
+                DataType.MEDIUMBLOB,
+                DataType.BINARY,
+                DataType.LONGBLOB,
+            },
+            sqlalchemy.VARBINARY: {DataType.VARBINARY},
+            CustomTypes.ARRAY.value: {DataType.ARRAY},
+            sqa_types.SQAMap: {DataType.MAP},
+            sqa_types.SQAStruct: {DataType.STRUCT},
+            sqa_types.SQAUnion: {DataType.UNION},
+            sqa_types.SQASet: {DataType.SET},
+            sqa_types.SQASGeography: {DataType.GEOGRAPHY},
+            sqlalchemy.Enum: {DataType.ENUM},
+            sqlalchemy.JSON: {DataType.JSON},
+            CustomTypes.UUID.value: {DataType.UUID},
+            CustomTypes.BYTEA.value: {DataType.BYTEA},
+            sqlalchemy.NVARCHAR: {DataType.NTEXT},
+            CustomTypes.IMAGE.value: {DataType.IMAGE},
+            CustomTypes.IP.value: {DataType.IPV6, DataType.IPV4},
+            CustomTypes.SQADATETIMERANGE.value: {DataType.DATETIMERANGE},
+        }

metadata/profiler/orm/converter/converter_registry.py

@@ -19,8 +19,10 @@ from metadata.generated.schema.entity.services.databaseService import (
 )
 from metadata.profiler.orm.converter.bigquery.converter import BigqueryMapTypes
 from metadata.profiler.orm.converter.common import CommonMapTypes
+from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
 from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
 
 converter_registry = defaultdict(lambda: CommonMapTypes)
 converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
 converter_registry[DatabaseServiceType.Snowflake] = SnowflakeMapTypes
+converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes