openmetadata-ingestion 1.3.4.0__py3-none-any.whl → 1.4.0.0rc2__py3-none-any.whl

metadata/profiler/metrics/static/distinct_count.py

@@ -14,9 +14,11 @@ Distinct Count Metric definition
 """
 # pylint: disable=duplicate-code
 
+import json
 
 from sqlalchemy import column, distinct, func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.count import CountFn
 from metadata.utils.logger import profiler_logger
@@ -33,7 +35,7 @@ class DistinctCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "distinctCount"
+        return MetricType.distinctCount.value
 
     @property
     def metric_type(self):
@@ -57,7 +59,14 @@ class DistinctCount(StaticMetric):
             counter = Counter()
             for df in dfs:
                 df_col_value = df[self.col.name].dropna().to_list()
-                counter.update(df_col_value)
+                try:
+                    counter.update(df_col_value)
+                except TypeError as err:
+                    if isinstance(df_col_value, list):
+                        for value in df_col_value:
+                            counter.update([json.dumps(value)])
+                    else:
+                        raise err
             return len(counter.keys())
         except Exception as err:
             logger.debug(

metadata/profiler/metrics/static/ilike_count.py

@@ -16,6 +16,7 @@ ILIKE Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -35,7 +36,7 @@ class ILikeCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "iLikeCount"
+        return MetricType.iLikeCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/like_count.py

@@ -16,6 +16,7 @@ Like Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -35,7 +36,7 @@ class LikeCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "likeCount"
+        return MetricType.likeCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/max.py

@@ -12,14 +12,17 @@
 """
 Max Metric definition
 """
-# pylint: disable=duplicate-code
-
+from functools import partial
+from typing import Callable, Optional
 
 from sqlalchemy import TIME, column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
-from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import DataType, Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
     FLOAT_SET,
@@ -29,6 +32,8 @@ from metadata.profiler.orm.registry import (
     is_quantifiable,
 )
 
+# pylint: disable=duplicate-code
+
 
 class MaxFn(GenericFunction):
     name = __qualname__
@@ -77,7 +82,7 @@ class Max(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "max"
+        return MetricType.max.value
 
     @_label
     def fn(self):
@@ -90,9 +95,23 @@ class Max(StaticMetric):
 
     def df_fn(self, dfs=None):
         """pandas function"""
+        import pandas as pd
+
         if is_quantifiable(self.col.type):
             return max((df[self.col.name].max() for df in dfs))
+
         if is_date_time(self.col.type):
-            max_ = max((df[self.col.name].max() for df in dfs))
-            return int(max_.timestamp() * 1000)
-        return 0
+            max_ = None
+            if self.col.type in {DataType.DATETIME, DataType.DATE}:
+                max_ = max((pd.to_datetime(df[self.col.name]).max() for df in dfs))
+                return None if pd.isnull(max_) else int(max_.timestamp() * 1000)
+            elif self.col.type == DataType.TIME:
+                max_ = max((pd.to_timedelta(df[self.col.name]).max() for df in dfs))
+                return None if pd.isnull(max_) else max_.seconds
+        return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.max, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/max_length.py

@@ -17,6 +17,7 @@ MAX_LENGTH Metric definition
 
 from sqlalchemy import column, func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class MaxLength(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "maxLength"
+        return MetricType.maxLength.value
 
     @property
     def metric_type(self):
@@ -66,7 +67,7 @@ class MaxLength(StaticMetric):
             max_length_list = []
 
             for df in dfs:
-                if any(df[self.col.name]):
+                if any(df[self.col.name].dropna()):
                     max_length_list.append(
                         length_vectorize_func(
                             df[self.col.name].dropna().astype(str)

metadata/profiler/metrics/static/mean.py

@@ -12,16 +12,17 @@
 """
 AVG Metric definition
 """
-# pylint: disable=duplicate-code
-
-
-from typing import List, cast
+from functools import partial
+from typing import Callable, List, Optional, cast
 
 from sqlalchemy import column, func
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
-from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
     FLOAT_SET,
@@ -32,6 +33,9 @@ from metadata.profiler.orm.registry import (
 )
 from metadata.utils.logger import profiler_logger
 
+# pylint: disable=duplicate-code
+
+
 logger = profiler_logger()
 
 
@@ -85,7 +89,7 @@ class Mean(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "mean"
+        return MetricType.mean.value
 
     @property
     def metric_type(self):
@@ -115,25 +119,23 @@ class Mean(StaticMetric):
 
         means = []
         weights = []
-
-        if is_quantifiable(self.col.type):
-            for df in dfs:
-                mean = df[self.col.name].mean()
-                if not pd.isnull(mean):
-                    means.append(mean)
-                    weights.append(df[self.col.name].count())
-
-        if is_concatenable(self.col.type):
-            length_vectorize_func = vectorize(len)
-            for df in dfs:
+        length_vectorize_func = vectorize(len)
+        for df in dfs:
+            processed_df = df[self.col.name].dropna()
+            try:
                 mean = None
-                if any(df[self.col.name]):
-                    mean = length_vectorize_func(
-                        df[self.col.name].dropna().astype(str)
-                    ).mean()
+                if is_quantifiable(self.col.type):
+                    mean = processed_df.mean()
+                if is_concatenable(self.col.type):
+                    mean = length_vectorize_func(processed_df.astype(str)).mean()
                 if not pd.isnull(mean):
                     means.append(mean)
-                    weights.append(df[self.col.name].dropna().count())
+                    weights.append(processed_df.count())
+            except Exception as err:
+                logger.debug(
+                    f"Error while computing mean for column {self.col.name}: {err}"
+                )
+                return None
 
         if means:
             return average(means, weights=weights)
@@ -142,3 +144,9 @@ class Mean(StaticMetric):
             f"Don't know how to process type {self.col.type} when computing MEAN"
         )
         return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.mean, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/min.py

@@ -12,13 +12,17 @@
 """
 Min Metric definition
 """
-# pylint: disable=duplicate-code
+from functools import partial
+from typing import Callable, Optional
 
 from sqlalchemy import TIME, column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
-from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import DataType, Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import CACHE, StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
     FLOAT_SET,
@@ -28,6 +32,8 @@ from metadata.profiler.orm.registry import (
     is_quantifiable,
 )
 
+# pylint: disable=duplicate-code
+
 
 class MinFn(GenericFunction):
     name = __qualname__
@@ -76,7 +82,7 @@ class Min(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "min"
+        return MetricType.min.value
 
     @_label
     def fn(self):
@@ -90,9 +96,22 @@ class Min(StaticMetric):
 
     def df_fn(self, dfs=None):
         """pandas function"""
+        import pandas as pd
+
         if is_quantifiable(self.col.type):
             return min((df[self.col.name].min() for df in dfs))
         if is_date_time(self.col.type):
-            min_ = min((df[self.col.name].min() for df in dfs))
-            return int(min_.timestamp() * 1000)
-        return 0
+            min_ = None
+            if self.col.type in {DataType.DATETIME, DataType.DATE}:
+                min_ = min((pd.to_datetime(df[self.col.name]).min() for df in dfs))
+                return None if pd.isnull(min_) else int(min_.timestamp() * 1000)
+            elif self.col.type == DataType.TIME:
+                min_ = min((pd.to_timedelta(df[self.col.name]).min() for df in dfs))
+                return None if pd.isnull(min_) else min_.seconds
+        return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.min, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/min_length.py

@@ -17,6 +17,7 @@ MIN_LENGTH Metric definition
 
 from sqlalchemy import column, func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class MinLength(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "minLength"
+        return MetricType.minLength.value
 
     @property
     def metric_type(self):
@@ -67,7 +68,7 @@ class MinLength(StaticMetric):
             min_length_list = []
 
             for df in dfs:
-                if any(df[self.col.name]):
+                if any(df[self.col.name].dropna()):
                     min_length_list.append(
                         length_vectorize_func(
                             df[self.col.name].dropna().astype(str)

metadata/profiler/metrics/static/not_like_count.py

@@ -16,6 +16,7 @@ Like Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -35,7 +36,7 @@ class NotLikeCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "notLikeCount"
+        return MetricType.notLikeCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/not_regexp_match_count.py

@@ -16,6 +16,7 @@ Regex Count Metric definition
 
 from sqlalchemy import case, column, not_
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class NotRegexCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "notRegexCount"
+        return MetricType.notRegexCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/null_count.py

@@ -17,6 +17,7 @@ Null Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 
@@ -38,7 +39,7 @@ class NullCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "nullCount"
+        return MetricType.nullCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/null_missing_count.py

@@ -0,0 +1,73 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Null Count Metric definition
+"""
+# pylint: disable=duplicate-code
+
+
+from sqlalchemy import case, column
+
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.orm.functions.sum import SumFn
+
+
+class NullMissingCount(StaticMetric):
+    """
+    NULL + Empty COUNT Metric
+
+    Given a column, return the null count.
+
+    We are building a CASE WHEN structure:
+    ```
+    SUM(
+        CASE is not null THEN 1
+        ELSE 0
+    )
+    ```
+    """
+
+    @classmethod
+    def name(cls):
+        """
+        Returns the name of the metric.
+        """
+        return MetricType.nullCount.value
+
+    @property
+    def metric_type(self):
+        """
+        Returns the type of the metric.
+        """
+        return int
+
+    @_label
+    def fn(self):
+        """
+        Returns the SQLAlchemy function for calculating the metric.
+        """
+        return SumFn(
+            case(
+                [
+                    (column(self.col.name, self.col.type).is_(None), 1),
+                    (column(self.col.name, self.col.type).__eq__(""), 1),
+                ],
+                else_=0,
+            )
+        )
+
+    def df_fn(self, dfs=None):
+        """
+        Returns the pandas function for calculating the metric.
+        """
+        return sum(df[self.col.name].isnull().sum() for df in dfs)

metadata/profiler/metrics/static/regexp_match_count.py

@@ -16,6 +16,7 @@ Regex Count Metric definition
 
 from sqlalchemy import case, column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable
@@ -36,7 +37,7 @@ class RegexCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "regexCount"
+        return MetricType.regexCount.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/row_count.py

@@ -12,11 +12,13 @@
 """
 Table Count Metric definition
 """
-# pylint: disable=duplicate-code
-
+from typing import Callable
 
 from sqlalchemy import func
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
 from metadata.profiler.metrics.core import StaticMetric, _label
 
 
@@ -29,7 +31,7 @@ class RowCount(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "rowCount"
+        return MetricType.rowCount.value
 
     @classmethod
     def is_col_metric(cls) -> bool:
@@ -50,3 +52,7 @@ class RowCount(StaticMetric):
     def df_fn(self, dfs=None):
         """pandas function"""
         return sum(len(df.index) for df in dfs)
+
+    @classmethod
+    def nosql_fn(cls, client: NoSQLAdaptor) -> Callable[[Table], int]:
+        return client.item_count

metadata/profiler/metrics/static/stddev.py

@@ -21,6 +21,7 @@ from sqlalchemy import column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import FunctionElement
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import CACHE, StaticMetric, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.registry import (
@@ -103,7 +104,7 @@ class StdDev(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "stddev"
+        return MetricType.stddev.value
 
     @property
     def metric_type(self):

metadata/profiler/metrics/static/sum.py

@@ -12,15 +12,21 @@
 """
 SUM Metric definition
 """
-# pylint: disable=duplicate-code
+from functools import partial
+from typing import Callable, Optional
 
 from sqlalchemy import column
 
-from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
+from metadata.generated.schema.entity.data.table import Table
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.metrics.core import StaticMetric, T, _label
 from metadata.profiler.orm.functions.length import LenFn
 from metadata.profiler.orm.functions.sum import SumFn
 from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
 
+# pylint: disable=duplicate-code
+
 
 class Sum(StaticMetric):
     """
@@ -33,7 +39,7 @@ class Sum(StaticMetric):
 
     @classmethod
     def name(cls):
-        return "sum"
+        return MetricType.sum.value
 
     @_label
     def fn(self):
@@ -50,5 +56,17 @@ class Sum(StaticMetric):
         """pandas function"""
 
         if is_quantifiable(self.col.type):
-            return sum(df[self.col.name].sum() for df in dfs)
+            try:
+                return sum(df[self.col.name].sum() for df in dfs)
+            except (TypeError, ValueError):
+                try:
+                    return sum(df[self.col.name].astype(float).sum() for df in dfs)
+                except Exception:
+                    return None
         return None
+
+    def nosql_fn(self, adaptor: NoSQLAdaptor) -> Callable[[Table], Optional[T]]:
+        """nosql function"""
+        if is_quantifiable(self.col.type):
+            return partial(adaptor.sum, column=self.col)
+        return lambda table: None

metadata/profiler/metrics/static/unique_count.py

@@ -12,11 +12,13 @@
 """
 Unique Count Metric definition
 """
+import json
 from typing import Optional
 
 from sqlalchemy import column, func
 from sqlalchemy.orm import DeclarativeMeta, Session
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import QueryMetric
 from metadata.profiler.orm.functions.unique_count import _unique_count_query_mapper
 from metadata.profiler.orm.registry import NOT_COMPUTE
@@ -34,7 +36,7 @@ class UniqueCount(QueryMetric):
 
     @classmethod
     def name(cls):
-        return "uniqueCount"
+        return MetricType.uniqueCount.value
 
     @property
     def metric_type(self):
@@ -59,8 +61,8 @@ class UniqueCount(QueryMetric):
         unique_count_query = _unique_count_query_mapper[session.bind.dialect.name](
             col, session, sample
         )
-        only_once_cte = unique_count_query.cte("only_once")
-        return session.query(func.count().label(self.name())).select_from(only_once_cte)
+        only_once_sub = unique_count_query.subquery("only_once")
+        return session.query(func.count().label(self.name())).select_from(only_once_sub)
 
     def df_fn(self, dfs=None):
         """
@@ -72,7 +74,14 @@ class UniqueCount(QueryMetric):
             counter = Counter()
             for df in dfs:
                 df_col_value = df[self.col.name].dropna().to_list()
-                counter.update(df_col_value)
+                try:
+                    counter.update(df_col_value)
+                except TypeError as err:
+                    if isinstance(df_col_value, list):
+                        for value in df_col_value:
+                            counter.update([json.dumps(value)])
+                    else:
+                        raise err
             return len([key for key, value in counter.items() if value == 1])
         except Exception as err:
             logger.debug(

metadata/profiler/metrics/system/system.py

@@ -20,6 +20,7 @@ from typing import Dict, List, Optional
 from sqlalchemy import text
 from sqlalchemy.orm import DeclarativeMeta, Session
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
@@ -430,7 +431,7 @@ class System(SystemMetric):
 
     @classmethod
     def name(cls):
-        return "system"
+        return MetricType.system.value
 
     def _manage_cache(self, max_size_in_bytes: int = MAX_SIZE_IN_BYTES) -> None:
         """manage cache and clears it if it exceeds the max size

metadata/profiler/metrics/window/first_quartile.py

@@ -18,6 +18,7 @@ from typing import List, cast
 
 from sqlalchemy import column
 
+from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
 from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
@@ -38,7 +39,7 @@ class FirstQuartile(StaticMetric, PercentilMixin):
 
     @classmethod
     def name(cls):
-        return "firstQuartile"
+        return MetricType.firstQuartile.value
 
     @classmethod
     def is_window_metric(cls):