openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0__py3-none-any.whl

metadata/profiler/orm/converter/redshift/converter.py

@@ -0,0 +1,48 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Converter logic to transform an OpenMetadata Table Entity for Redshift
+to an SQLAlchemy ORM class.
+"""
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.entity.services import databaseService
+from metadata.profiler.orm.converter.common import CommonMapTypes
+
+
+class RedshiftMapTypes(CommonMapTypes):
+    """Redshift mapper, inherits from CommonMapTypes"""
+
+    def __init__(self) -> None:
+        self._TYPE_MAP.update({DataType.GEOMETRY: DataType.GEOMETRY.value})
+
+    def return_custom_type(self, col: Column, table_service_type):
+        if (
+            table_service_type == databaseService.DatabaseServiceType.Redshift
+            and col.dataType == DataType.GEOMETRY
+        ):
+            # pylint: disable=import-outside-toplevel
+            from sqlalchemy_redshift.dialect import GEOMETRY
+
+            return GEOMETRY
+        return super().return_custom_type(col, table_service_type)
+
+    @staticmethod
+    def map_sqa_to_om_types() -> dict:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+        from sqlalchemy_redshift.dialect import GEOMETRY
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            GEOMETRY: DataType.GEOMETRY,
+        }

metadata/profiler/orm/converter/snowflake/converter.py

@@ -22,6 +22,8 @@ from metadata.profiler.orm.registry import CustomTypes
 
 
 class SnowflakeMapTypes(CommonMapTypes):
+    """Snowflake mapper, inherits from CommonMapTypes"""
+
     def __init__(self) -> None:
         self._TYPE_MAP.update({DataType.BINARY: CustomTypes.BYTES.value})
 
@@ -35,3 +37,14 @@ class SnowflakeMapTypes(CommonMapTypes):
 
             return VARIANT
         return super().return_custom_type(col, table_service_type)
+
+    @staticmethod
+    def map_sqa_to_om_types() -> dict:
+        """returns an ORM type"""
+        # pylint: disable=import-outside-toplevel
+        from snowflake.sqlalchemy import VARIANT
+
+        return {
+            **CommonMapTypes.map_sqa_to_om_types(),
+            VARIANT: DataType.JSON,
+        }

metadata/profiler/orm/functions/conn_test.py

@@ -50,7 +50,7 @@ def _(*_, **__):
 @compiles(ConnTestFn, Dialects.IbmDbSa)
 @compiles(ConnTestFn, Dialects.Ibmi)
 def _(*_, **__):
-    return "SELECT 42 FROM SYSIBM.SYSDUMMY1;"
+    return "SELECT 42 FROM SYSIBM.SYSDUMMY1"
 
 
 @compiles(ConnTestFn, Dialects.Hana)

metadata/profiler/orm/functions/sum.py

@@ -32,11 +32,13 @@ def _(element, compiler, **kw):
     return f"SUM(CAST({proc} AS BIGINT))"
 
 
+@compiles(SumFn, Dialects.Athena)
 @compiles(SumFn, Dialects.Trino)
+@compiles(SumFn, Dialects.Presto)
 def _(element, compiler, **kw):
-    """Cast to BIGINT to address cannot cast nan to bigint"""
+    """Cast to DECIMAL to address cannot cast nan to bigint"""
     proc = compiler.process(element.clauses, **kw)
-    return f"SUM(TRY_CAST({proc} AS BIGINT))"
+    return f"SUM(TRY_CAST({proc} AS DECIMAL))"
 
 
 @compiles(SumFn, Dialects.BigQuery)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -194,7 +194,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("object_name").label("table_name"),
                     Column("created"),
                 ],
-                self._build_table("dba_objects", None),
+                self._build_table("all_objects", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("object_name")) == self.table_name.lower(),
@@ -231,6 +231,8 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
@@ -260,6 +262,8 @@ class ClickHouseTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
@@ -306,6 +310,8 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
@@ -337,6 +343,8 @@ class BigQueryTableMetricComputer(BaseTableMetricComputer):
             where_clause,
         )
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
@@ -366,6 +374,8 @@ class MySQLTableMetricComputer(BaseTableMetricComputer):
         )
 
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
@@ -400,6 +410,8 @@ class RedshiftTableMetricComputer(BaseTableMetricComputer):
             columns, self._build_table("svv_table_info", "pg_catalog"), where_clause
         )
         res = self.runner._session.execute(query).first()
+        if not res:
+            return None
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):

metadata/profiler/orm/registry.py

@@ -92,6 +92,7 @@ NOT_COMPUTE = {
     sqa_types.SQASet.__name__,
     sqa_types.SQAUnion.__name__,
     sqa_types.SQASGeography.__name__,
+    DataType.GEOMETRY.value,
     DataType.ARRAY.value,
     DataType.JSON.value,
     CustomTypes.ARRAY.value.__name__,
@@ -114,8 +115,6 @@ QUANTIFIABLE_SET = {
 
 CONCATENABLE_SET = {DataType.STRING.value, DataType.TEXT.value}
 
-DATATIME_SET = {DataType.DATETIME.value}
-
 
 # Now, let's define some helper methods to identify
 # the nature of an SQLAlchemy type
@@ -138,7 +137,11 @@ def is_date_time(_type) -> bool:
     Check if sqlalchemy _type is derived from Date, Time or DateTime Type
     """
     if isinstance(_type, DataType):
-        return _type.value in DATATIME_SET
+        return _type.value in {
+            DataType.DATETIME.value,
+            DataType.TIME.value,
+            DataType.DATE.value,
+        }
     return (
         issubclass(_type.__class__, Date)
         or issubclass(_type.__class__, Time)

metadata/profiler/processor/core.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 
 import traceback
 from datetime import datetime, timezone
-from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, cast
+from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type
 
 from pydantic import ValidationError
 from sqlalchemy import Column
@@ -25,6 +25,9 @@ from sqlalchemy.orm import DeclarativeMeta
 from metadata.generated.schema.api.data.createTableProfile import (
     CreateTableProfileRequest,
 )
+from metadata.generated.schema.configuration.profilerConfiguration import (
+    ProfilerConfiguration,
+)
 from metadata.generated.schema.entity.data.table import (
     ColumnName,
     ColumnProfile,
@@ -33,6 +36,7 @@ from metadata.generated.schema.entity.data.table import (
     TableData,
     TableProfile,
 )
+from metadata.generated.schema.settings.settings import Settings
 from metadata.generated.schema.tests.customMetric import (
     CustomMetric as CustomMetricEntity,
 )
@@ -40,17 +44,15 @@ from metadata.profiler.api.models import ProfilerResponse, ThreadPoolMetrics
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
 from metadata.profiler.metrics.core import (
     ComposedMetric,
-    CustomMetric,
     HybridMetric,
     MetricTypes,
     QueryMetric,
     StaticMetric,
-    SystemMetric,
     TMetric,
 )
-from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.metrics.static.row_count import RowCount
 from metadata.profiler.orm.registry import NOT_COMPUTE
+from metadata.profiler.processor.metric_filter import MetricFilter
 from metadata.profiler.processor.sample_data_handler import upload_sample_data
 from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
 from metadata.utils.execution_time_tracker import calculate_execution_time
@@ -76,12 +78,15 @@ class Profiler(Generic[TMetric]):
     - A tuple of metrics, from which we will construct queries.
     """
 
+    # pylint: disable=too-many-instance-attributes
+
     def __init__(
         self,
         *metrics: Type[TMetric],
         profiler_interface: ProfilerInterface,
         include_columns: Optional[List[ColumnProfilerConfig]] = None,
         exclude_columns: Optional[List[str]] = None,
+        global_profiler_configuration: Optional[Settings] = None,
     ):
         """
         :param metrics: Metrics to run. We are receiving the uninitialized classes
@@ -90,7 +95,11 @@ class Profiler(Generic[TMetric]):
         :param ignore_cols: List of columns to ignore when computing the profile
         :param profile_sample: % of rows to use for sampling column metrics
         """
-
+        self.global_profiler_configuration: Optional[ProfilerConfiguration] = (
+            global_profiler_configuration.config_value
+            if global_profiler_configuration
+            else None
+        )
         self.profiler_interface = profiler_interface
         self.source_config = self.profiler_interface.source_config
         self.include_columns = include_columns
@@ -99,6 +108,13 @@ class Profiler(Generic[TMetric]):
         self._profile_date = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
         self.profile_sample_config = self.profiler_interface.profile_sample_config
 
+        self.metric_filter = MetricFilter(
+            metrics=self.metrics,
+            global_profiler_config=self.global_profiler_configuration,
+            table_profiler_config=self.profiler_interface.table_entity.tableProfilerConfig,
+            column_profiler_config=self.include_columns,
+        )
+
         self.validate_composed_metric()
 
         # Initialize profiler results
@@ -175,12 +191,6 @@ class Profiler(Generic[TMetric]):
             return {include_col.columnName for include_col in self.include_columns}
         return {}
 
-    def _filter_metrics(self, _type: Type[TMetric]) -> List[Type[TMetric]]:
-        """
-        Filter metrics by type
-        """
-        return [metric for metric in self.metrics if issubclass(metric, _type)]
-
     def _check_profile_and_handle(
         self, profile: CreateTableProfileRequest
     ) -> CreateTableProfileRequest:
@@ -196,7 +206,10 @@ class Profiler(Generic[TMetric]):
             CreateTableProfileRequest:
         """
         for attrs, val in profile.tableProfile:
-            if attrs not in {"timestamp", "profileSample", "profileSampleType"} and val:
+            if (
+                attrs not in {"timestamp", "profileSample", "profileSampleType"}
+                and val is not None
+            ):
                 return
 
         for col_element in profile.columnProfile:
@@ -208,76 +221,6 @@ class Profiler(Generic[TMetric]):
             f"No profile data computed for {self.profiler_interface.table_entity.fullyQualifiedName.__root__}"
         )
 
-    @property
-    def static_metrics(self) -> List[Type[StaticMetric]]:
-        return self._filter_metrics(StaticMetric)
-
-    @property
-    def composed_metrics(self) -> List[Type[ComposedMetric]]:
-        return self._filter_metrics(ComposedMetric)
-
-    @property
-    def custom_metrics(self) -> List[Type[CustomMetric]]:
-        return self._filter_metrics(CustomMetric)
-
-    @property
-    def query_metrics(self) -> List[Type[QueryMetric]]:
-        return self._filter_metrics(QueryMetric)
-
-    @property
-    def system_metrics(self) -> List[Type[SystemMetric]]:
-        return self._filter_metrics(SystemMetric)
-
-    @property
-    def hybrid_metric(self) -> List[Type[HybridMetric]]:
-        return self._filter_metrics(HybridMetric)
-
-    def get_col_metrics(
-        self, metrics: List[Type[TMetric]], column: Optional[Column] = None
-    ) -> List[Type[TMetric]]:
-        """
-        Filter list of metrics for column metrics with allowed types
-        """
-
-        if column is None:
-            return [metric for metric in metrics if metric.is_col_metric()]
-
-        if (
-            self.profiler_interface.table_entity.tableProfilerConfig
-            and self.profiler_interface.table_entity.tableProfilerConfig.includeColumns
-        ) or (self.include_columns):
-            # include_columns is set from the `tableConfig` of the `ProfilerProcessorConfig` in the CLI config
-            # while `self.profiler_interface.table_entity.tableProfilerConfig.includeColumns` is set from the entity
-            # definition in the metadata service. This gets set either from the UI or the profiler entity page. Config
-            # ran from the CLI takes precedence over the entity definition.
-            columns = (
-                self.include_columns
-                if self.include_columns
-                else self.profiler_interface.table_entity.tableProfilerConfig.includeColumns
-            )
-            columns = cast(List[ColumnProfilerConfig], columns)
-            metric_names = next(
-                (
-                    include_columns.metrics
-                    for include_columns in columns
-                    if include_columns.columnName == column.name
-                ),
-                None,
-            )
-
-            if metric_names:
-                metric_names = {
-                    mtrc.lower() for mtrc in metric_names
-                }  # case insensitice
-                metrics = [
-                    Metric.value
-                    for Metric in Metrics
-                    if Metric.value.name().lower() in metric_names
-                    and Metric.value in metrics
-                ]
-
-        return [metric for metric in metrics if metric.is_col_metric()]
-
     def get_custom_metrics(
         self, column_name: Optional[str] = None
     ) -> Optional[List[CustomMetricEntity]]:
@@ -317,7 +260,7 @@ class Profiler(Generic[TMetric]):
         `required_metrics` attr
         """
         names = {metric.name() for metric in self.metrics}
-        for metric in self.composed_metrics:
+        for metric in self.metric_filter.composed_metrics:
             if not set(metric.required_metrics()).issubset(names):
                 raise MissingMetricException(
                     f"We need {metric.required_metrics()} for {metric.name}, but only got {names} in the profiler"
@@ -339,7 +282,9 @@ class Profiler(Generic[TMetric]):
             )
             return
 
-        for metric in self.get_col_metrics(self.composed_metrics):
+        for metric in self.metric_filter.get_column_metrics(
+            ComposedMetric, col, self.profiler_interface.table_entity.serviceType
+        ):
             # Composed metrics require the results as an argument
             logger.debug(f"Running composed metric {metric.name()} for {col.name}")
 
@@ -364,7 +309,9 @@ class Profiler(Generic[TMetric]):
                 "We do not have any results to base our Hybrid Metrics. Stopping!"
             )
             return
-        for metric in self.get_col_metrics(self.hybrid_metric, col):
+        for metric in self.metric_filter.get_column_metrics(
+            HybridMetric, col, self.profiler_interface.table_entity.serviceType
+        ):
             logger.debug(f"Running hybrid metric {metric.name()} for {col.name}")
             self._column_results[col.name][
                 metric.name()
@@ -380,7 +327,7 @@ class Profiler(Generic[TMetric]):
         metrics = []
         table_metrics = [
             metric
-            for metric in self.static_metrics
+            for metric in self.metric_filter.static_metrics
             if (not metric.is_col_metric() and not metric.is_system_metrics())
         ]
 
@@ -414,7 +361,7 @@ class Profiler(Generic[TMetric]):
 
     def _prepare_system_metrics(self) -> List:
         """prepare system metrics"""
-        system_metrics = self.system_metrics
+        system_metrics = self.metric_filter.system_metrics
 
         if system_metrics:
             return [
@@ -431,17 +378,21 @@ class Profiler(Generic[TMetric]):
 
     def _prepare_column_metrics(self) -> List:
         """prepare column metrics"""
+        column_metrics_for_thread_pool = []
         columns = [
             column
             for column in self.columns
             if column.type.__class__.__name__ not in NOT_COMPUTE
         ]
-        column_metrics_for_thread_pool = []
         static_metrics = [
             ThreadPoolMetrics(
                 metrics=[
                     metric
-                    for metric in self.get_col_metrics(self.static_metrics, column)
+                    for metric in self.metric_filter.get_column_metrics(
+                        StaticMetric,
+                        column,
+                        self.profiler_interface.table_entity.serviceType,
+                    )
                     if not metric.is_window_metric()
                 ],
                 metric_type=MetricTypes.Static,
@@ -458,13 +409,19 @@ class Profiler(Generic[TMetric]):
                 table=self.table,
             )
             for column in columns
-            for metric in self.get_col_metrics(self.query_metrics, column)
+            for metric in self.metric_filter.get_column_metrics(
+                QueryMetric, column, self.profiler_interface.table_entity.serviceType
+            )
         ]
         window_metrics = [
             ThreadPoolMetrics(
                 metrics=[
                     metric
-                    for metric in self.get_col_metrics(self.static_metrics, column)
+                    for metric in self.metric_filter.get_column_metrics(
+                        StaticMetric,
+                        column,
+                        self.profiler_interface.table_entity.serviceType,
+                    )
                     if metric.is_window_metric()
                 ],
                 metric_type=MetricTypes.Window,
@@ -628,12 +585,16 @@ class Profiler(Generic[TMetric]):
                 rowCount=self._table_results.get(RowCount.name()),
                 createDateTime=self._table_results.get("createDateTime"),
                 sizeInByte=self._table_results.get("sizeInBytes"),
-                profileSample=self.profile_sample_config.profile_sample
-                if self.profile_sample_config
-                else None,
-                profileSampleType=self.profile_sample_config.profile_sample_type
-                if self.profile_sample_config
-                else None,
+                profileSample=(
+                    self.profile_sample_config.profile_sample
+                    if self.profile_sample_config
+                    else None
+                ),
+                profileSampleType=(
+                    self.profile_sample_config.profile_sample_type
+                    if self.profile_sample_config
+                    else None
+                ),
                 customMetrics=self._table_results.get("customMetrics"),
             )
 

metadata/profiler/processor/default.py

@@ -16,6 +16,9 @@ from typing import List, Optional
 
 from sqlalchemy.orm import DeclarativeMeta
 
+from metadata.generated.schema.configuration.profilerConfiguration import (
+    ProfilerConfiguration,
+)
 from metadata.generated.schema.entity.data.table import ColumnProfilerConfig
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -73,6 +76,7 @@ class DefaultProfiler(Profiler):
         profiler_interface: ProfilerInterface,
         include_columns: Optional[List[ColumnProfilerConfig]] = None,
         exclude_columns: Optional[List[str]] = None,
+        global_profiler_configuration: Optional[ProfilerConfiguration] = None,
     ):
         _metrics = get_default_metrics(
             table=profiler_interface.table, ometa_client=profiler_interface.ometa_client
@@ -83,4 +87,5 @@ class DefaultProfiler(Profiler):
             profiler_interface=profiler_interface,
             include_columns=include_columns,
             exclude_columns=exclude_columns,
+            global_profiler_configuration=global_profiler_configuration,
         )

metadata/profiler/processor/handle_partition.py

@@ -19,7 +19,7 @@ from typing import List
 from sqlalchemy import Column, text
 
 from metadata.generated.schema.entity.data.table import (
-    PartitionIntervalType,
+    PartitionIntervalTypes,
     PartitionProfilerConfig,
 )
 from metadata.profiler.orm.functions.modulo import ModuloFn
@@ -52,13 +52,13 @@ def build_partition_predicate(
         _type_: _description_
     """
     partition_field = partition_details.partitionColumnName
-    if partition_details.partitionIntervalType == PartitionIntervalType.COLUMN_VALUE:
+    if partition_details.partitionIntervalType == PartitionIntervalTypes.COLUMN_VALUE:
         return get_value_filter(
             Column(partition_field),
             partition_details.partitionValues,
         )
 
-    if partition_details.partitionIntervalType == PartitionIntervalType.INTEGER_RANGE:
+    if partition_details.partitionIntervalType == PartitionIntervalTypes.INTEGER_RANGE:
         return get_integer_range_filter(
             Column(partition_field),
             partition_details.partitionIntegerRangeStart,