openmetadata-ingestion 1.5.11.0__py3-none-any.whl → 1.6.0.0rc1__py3-none-any.whl

metadata/profiler/interface/profiler_interface.py

@@ -15,7 +15,7 @@ supporting sqlalchemy abstraction layer
 """
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
 from sqlalchemy import Column
 
@@ -29,6 +29,7 @@ from metadata.generated.schema.entity.data.databaseSchema import (
 )
 from metadata.generated.schema.entity.data.table import (
     PartitionProfilerConfig,
+    SystemProfile,
     Table,
     TableData,
 )
@@ -59,7 +60,9 @@ from metadata.profiler.api.models import (
 )
 from metadata.profiler.metrics.core import MetricTypes
 from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.metrics.system.system import System
 from metadata.profiler.processor.runner import QueryRunner
+from metadata.utils.collaborative_super import Root
 from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
 from metadata.utils.partition import get_partition_details
 from metadata.utils.ssl_manager import get_ssl_connection
@@ -84,11 +87,10 @@ class ProfilerProcessorStatus(Status):
 
 
 # pylint: disable=too-many-instance-attributes
-class ProfilerInterface(ABC):
+class ProfilerInterface(Root, ABC):
     """Protocol interface for the profiler processor"""
 
-    # pylint: disable=too-many-arguments,unused-argument
-    def __init__(
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         service_connection_config: Union[DatabaseConnection, DatalakeConnection],
         ometa_client: OpenMetadata,
@@ -135,6 +137,21 @@ class ProfilerInterface(ABC):
             MetricTypes.Custom.value: self._compute_custom_metrics,
         }
 
+        super().__init__(
+            service_connection_config=service_connection_config,
+            ometa_client=ometa_client,
+            entity=entity,
+            storage_config=storage_config,
+            profile_sample_config=profile_sample_config,
+            source_config=source_config,
+            sample_query=sample_query,
+            table_partition_config=table_partition_config,
+            thread_count=thread_count,
+            timeout_seconds=timeout_seconds,
+            sample_data_count=sample_data_count,
+            **kwargs,
+        )
+
     @abstractmethod
     def _get_sampler(self):
         """Get the sampler"""
@@ -142,7 +159,7 @@ class ProfilerInterface(ABC):
 
     # pylint: disable=too-many-locals
     @classmethod
-    def create(
+    def create(  # pylint: disable=too-many-arguments
         cls,
         entity: Table,
         database_schema: DatabaseSchema,
@@ -247,7 +264,7 @@ class ProfilerInterface(ABC):
             DatabaseSchemaProfilerConfig,
             DatabaseProfilerConfig,
             DatabaseAndSchemaConfig,
-        ]
+        ],
     ) -> Optional[DataStorageConfig]:
         if (
             config
@@ -328,6 +345,7 @@ class ProfilerInterface(ABC):
                     return ProfileSampleConfig(
                         profile_sample=config.profileSample,
                         profile_sample_type=config.profileSampleType,
+                        sampling_method_type=config.samplingMethodType,
                     )
             except AttributeError:
                 pass
@@ -458,11 +476,11 @@ class ProfilerInterface(ABC):
     @abstractmethod
     def _compute_system_metrics(
         self,
-        metrics: Metrics,
+        metrics: Type[System],
         runner,
         *args,
         **kwargs,
-    ):
+    ) -> List[SystemProfile]:
         """Get metrics"""
         raise NotImplementedError
 

metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py

@@ -24,11 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
 )
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.processor.runner import QueryRunner
-from metadata.profiler.source.mariadb.metrics.window.first_quartile import (
+from metadata.profiler.source.database.mariadb.metrics.window.first_quartile import (
     MariaDBFirstQuartile,
 )
-from metadata.profiler.source.mariadb.metrics.window.median import MariaDBMedian
-from metadata.profiler.source.mariadb.metrics.window.third_quartile import (
+from metadata.profiler.source.database.mariadb.metrics.window.median import (
+    MariaDBMedian,
+)
+from metadata.profiler.source.database.mariadb.metrics.window.third_quartile import (
     MariaDBThirdQuartile,
 )
 from metadata.utils.logger import profiler_interface_registry_logger

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -21,23 +21,28 @@ import threading
 import traceback
 from collections import defaultdict
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Type
 
 from sqlalchemy import Column, inspect, text
 from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
 from sqlalchemy.orm import scoped_session
 
-from metadata.generated.schema.entity.data.table import CustomMetricProfile, TableData
+from metadata.generated.schema.entity.data.table import (
+    CustomMetricProfile,
+    SystemProfile,
+    TableData,
+)
 from metadata.generated.schema.tests.customMetric import CustomMetric
 from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
 from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
 from metadata.profiler.api.models import ThreadPoolMetrics
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
-from metadata.profiler.metrics.core import MetricTypes
+from metadata.profiler.metrics.core import HybridMetric, MetricTypes
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.metrics.static.mean import Mean
 from metadata.profiler.metrics.static.stddev import StdDev
 from metadata.profiler.metrics.static.sum import Sum
+from metadata.profiler.metrics.system.system import System, SystemMetricsComputer
 from metadata.profiler.orm.functions.table_metric_computer import TableMetricComputer
 from metadata.profiler.orm.registry import Dialects
 from metadata.profiler.processor.metric_filter import MetricFilter
@@ -105,6 +110,13 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
 
         self._table = self._convert_table_to_orm_object(sqa_metadata)
         self.create_session()
+        self.system_metrics_computer = self.initialize_system_metrics_computer()
+
+    def initialize_system_metrics_computer(self) -> SystemMetricsComputer:
+        """Initialize system metrics computer. Override this if you want to use a metric source with
+        state or other dependencies.
+        """
+        return SystemMetricsComputer()
 
     def create_session(self):
         self.session_factory = self._session_factory()
@@ -174,7 +186,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
         session,
         *args,
         **kwargs,
-    ):
+    ) -> Optional[Dict[str, Any]]:
         """Given a list of metrics, compute the given results
         and returns the values
 
@@ -363,12 +375,11 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
 
     def _compute_system_metrics(
         self,
-        metrics: Metrics,
+        metrics: Type[System],
         runner: QueryRunner,
-        session,
         *args,
         **kwargs,
-    ):
+    ) -> List[SystemProfile]:
         """Get system metric for tables
 
         Args:
@@ -379,13 +390,8 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
         Returns:
             dictionnary of results
         """
-        try:
-            rows = metrics().sql(session, conn_config=self.service_connection_config)
-            return rows
-        except Exception as exc:
-            msg = f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
-            handle_query_exception(msg, exc, session)
-        return None
+        logger.debug(f"Computing system metrics for {runner.table.__tablename__}")
+        return self.system_metrics_computer.get_system_metrics(table=runner.table)
 
     def _create_thread_safe_sampler(
         self,
@@ -442,7 +448,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                 sample,
             )
             row = None
-
             try:
                 row = self._get_metric_fn[metric_func.metric_type.value](
                     metric_func.metrics,
@@ -451,11 +456,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     column=metric_func.column,
                     sample=sample,
                 )
-                if row and isinstance(row, dict):
+                if isinstance(row, dict):
                     row = self._validate_nulls(row)
-
-                # System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
-                if row and metric_func.metric_type == MetricTypes.System:
+                if isinstance(row, list):
                     row = [
                         self._validate_nulls(r) if isinstance(r, dict) else r
                         for r in row
@@ -537,6 +540,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     logger.debug(traceback.format_exc())
                     logger.error(f"Operation was cancelled due to TimeoutError - {exc}")
                     raise concurrent.futures.TimeoutError
+                except KeyboardInterrupt:
+                    pool.shutdown39(wait=True, cancel_futures=True)
+                    raise
 
         return profile_results
 
@@ -576,14 +582,19 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             return None
 
     def get_hybrid_metrics(
-        self, column: Column, metric: Metrics, column_results: Dict, **kwargs
+        self,
+        column: Column,
+        metric: Type[HybridMetric],
+        column_results: Dict[str, Any],
+        **kwargs,
     ):
         """Given a list of metrics, compute the given results
         and returns the values
 
         Args:
             column: the column to compute the metrics against
-            metrics: list of metrics to compute
+            metric: metric to compute
+            column_results: results of the column
         Returns:
             dictionnary of results
         """

metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py

@@ -24,13 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
 )
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.processor.runner import QueryRunner
-from metadata.profiler.source.single_store.metrics.window.first_quartile import (
+from metadata.profiler.source.database.single_store.metrics.window.first_quartile import (
     SingleStoreFirstQuartile,
 )
-from metadata.profiler.source.single_store.metrics.window.median import (
+from metadata.profiler.source.database.single_store.metrics.window.median import (
     SingleStoreMedian,
 )
-from metadata.profiler.source.single_store.metrics.window.third_quartile import (
+from metadata.profiler.source.database.single_store.metrics.window.third_quartile import (
     SingleStoreThirdQuartile,
 )
 from metadata.utils.logger import profiler_interface_registry_logger

metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py

@@ -0,0 +1,147 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Interfaces with database for all database engine
+supporting sqlalchemy abstraction layer
+"""
+
+import threading
+from typing import Any, Dict, List, Set
+
+from more_itertools import partition
+from sqlalchemy import Column
+
+from metadata.generated.schema.entity.data.table import Table
+from metadata.mixins.sqalchemy.sqa_mixin import Root
+from metadata.profiler.interface.sqlalchemy.profiler_interface import (
+    SQAProfilerInterface,
+)
+from metadata.profiler.metrics.core import Metric
+from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.processor.runner import QueryRunner
+from metadata.utils.logger import profiler_interface_registry_logger
+
+logger = profiler_interface_registry_logger()
+thread_local = threading.local()
+
+
+class StoredStatisticsSource(Root):
+    def get_statistics_metrics(self) -> Set[Metrics]:
+        """Statistic metrics that are found in system tables. Different for each database."""
+        return set()
+
+    def get_column_statistics(
+        self, metric: List[Metrics], schema: str, table_name: Table, column: str
+    ) -> dict:
+        raise NotImplementedError(
+            "You used a connector that does not support using statistics tables."
+        )
+
+    def get_table_statistics(
+        self, metric: List[Metrics], schema: str, table_name: Table
+    ) -> dict:
+        raise NotImplementedError(
+            "You used a connector that does not support using statistics tables."
+        )
+
+
+class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
+    """
+    Interface to interact with registry supporting
+    sqlalchemy.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """Collaborative constructor"""
+        super().__init__(*args, **kwargs)
+
+    def _compute_static_metrics(
+        self,
+        metrics: List[Metrics],
+        runner: QueryRunner,
+        column,
+        session,
+        *args,
+        **kwargs,
+    ) -> dict:
+        result = {}
+        if self.source_config.useStatistics:
+            metrics, stat_metrics = map(
+                list,
+                partition(self.is_statistic_metric, metrics),
+            )
+            schema = runner.table.__table_args__["schema"]
+            table_name = runner.table.__tablename__
+            logger.debug(
+                "Getting statistics for column: %s.%s.%s",
+                schema,
+                table_name,
+                column.name,
+            )
+            result.update(
+                super().get_column_statistics(
+                    stat_metrics, schema, table_name, column.name
+                )
+            )
+        result.update(
+            super()._compute_static_metrics(
+                metrics,
+                runner,
+                column,
+                session,
+                *args,
+                **kwargs,
+            )
+        )
+        return result
+
+    def _compute_table_metrics(
+        self,
+        metrics: List[Metric],
+        runner: QueryRunner,
+        session,
+        *args,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        result = {}
+        if self.source_config.useStatistics:
+            metrics, stat_metrics = map(
+                list,
+                partition(self.is_statistic_metric, metrics),
+            )
+            schema = runner.table.__table_args__["schema"]
+            table_name = runner.table.__tablename__
+            logger.debug("Geting statistics for table: %s.%s", schema, table_name)
+            result.update(
+                super().get_table_statistics(stat_metrics, schema, table_name)
+            )
+        super_table_metrics = super()._compute_table_metrics(
+            metrics,
+            runner,
+            session,
+            *args,
+            **kwargs,
+        )
+        if super_table_metrics is not None:
+            result.update(super_table_metrics)
+        return result
+
+    def get_hybrid_metrics(
+        self, column: Column, metric: Metric, column_results: Dict, **kwargs
+    ):
+        # this metrics might have been computed in a previous step
+        return column_results.get(metric.name()) or super().get_hybrid_metrics(
+            column, metric, column_results, **kwargs
+        )
+
+    def is_statistic_metric(self, metric: Metric) -> bool:
+        return metric.name() in {m.name for m in super().get_statistics_metrics()}

metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py

@@ -19,10 +19,15 @@ from typing import List
 from sqlalchemy import func
 from sqlalchemy.exc import ProgrammingError
 
+from metadata.ingestion.source.database.trino.profiler.system_tables_profiler import (
+    TrinoStoredStatisticsSource,
+)
 from metadata.profiler.interface.sqlalchemy.profiler_interface import (
-    SQAProfilerInterface,
     handle_query_exception,
 )
+from metadata.profiler.interface.sqlalchemy.stored_statistics_profiler import (
+    ProfilerWithStatistics,
+)
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.orm.registry import FLOAT_SET
 from metadata.profiler.processor.runner import QueryRunner
@@ -31,7 +36,7 @@ from metadata.utils.logger import profiler_interface_registry_logger
 logger = profiler_interface_registry_logger()
 
 
-class TrinoProfilerInterface(SQAProfilerInterface):
+class TrinoProfilerInterface(ProfilerWithStatistics, TrinoStoredStatisticsSource):
     """
     Interface to interact with registry supporting
     sqlalchemy.