openmetadata-ingestion 1.1.4.0__py3-none-any.whl → 1.1.6.0__py3-none-any.whl

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -8,6 +8,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
+#  pylint: disable=arguments-differ
 
 """
 Interfaces with database for all database engine
@@ -40,7 +41,6 @@ from metadata.profiler.orm.functions.table_metric_construct import (
 from metadata.profiler.processor.runner import QueryRunner
 from metadata.profiler.processor.sampler.sampler_factory import sampler_factory_
 from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor
-from metadata.utils.dispatch import valuedispatch
 from metadata.utils.logger import profiler_interface_registry_logger
 
 logger = profiler_interface_registry_logger()
@@ -153,18 +153,8 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             handle_query_exception(msg, exc, session)
         return None
 
-    @valuedispatch
-    def _get_metrics(self, *args, **kwargs):
-        """Generic getter method for metrics. To be used with
-        specific dispatch methods
-        """
-        logger.warning("Could not get metric. No function registered.")
-
-    # pylint: disable=unused-argument
-    @_get_metrics.register(MetricTypes.Table.value)
-    def _(
+    def _compute_table_metrics(
         self,
-        metric_type: str,
         metrics: List[Metrics],
         runner: QueryRunner,
         session,
@@ -180,7 +170,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             dictionnary of results
         """
         # pylint: disable=protected-access
-
         try:
             dialect = runner._session.get_bind().dialect.name
             row = table_metric_construct_factory.construct(
@@ -201,15 +190,12 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             session.rollback()
             raise RuntimeError(exc)
 
-    # pylint: disable=unused-argument
-    @_get_metrics.register(MetricTypes.Static.value)
-    def _(
+    def _compute_static_metrics(
         self,
-        metric_type: str,
         metrics: List[Metrics],
         runner: QueryRunner,
+        column,
         session,
-        column: Column,
         *args,
         **kwargs,
     ):
@@ -247,16 +233,15 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             handle_query_exception(msg, exc, session)
         return None
 
-    # pylint: disable=unused-argument
-    @_get_metrics.register(MetricTypes.Query.value)
-    def _(
+    def _compute_query_metrics(
         self,
-        metric_type: str,
         metric: Metrics,
         runner: QueryRunner,
+        column,
         session,
-        column: Column,
         sample,
+        *args,
+        **kwargs,
     ):
         """Given a list of metrics, compute the given results
         and returns the values
@@ -267,6 +252,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
         Returns:
             dictionnary of results
         """
+
         try:
             col_metric = metric(column)
             metric_query = col_metric.query(sample=sample, session=session)
@@ -284,15 +270,12 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             handle_query_exception(msg, exc, session)
         return None
 
-    # pylint: disable=unused-argument
-    @_get_metrics.register(MetricTypes.Window.value)
-    def _(
+    def _compute_window_metrics(
         self,
-        metric_type: str,
         metrics: List[Metrics],
         runner: QueryRunner,
+        column,
         session,
-        column: Column,
         *args,
         **kwargs,
     ):
@@ -305,6 +288,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
         Returns:
             dictionnary of results
         """
+
         if not metrics:
             return None
         try:
@@ -327,11 +311,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             return dict(row)
         return None
 
-    @_get_metrics.register(MetricTypes.System.value)
-    def _(
+    def _compute_system_metrics(
         self,
-        metric_type: str,
-        metric: Metrics,
+        metrics: Metrics,
         runner: QueryRunner,
         session,
         *args,
@@ -348,7 +330,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             dictionnary of results
         """
         try:
-            rows = metric().sql(session, conn_config=self.service_connection_config)
+            rows = metrics().sql(session, conn_config=self.service_connection_config)
             return rows
         except Exception as exc:
             msg = f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
@@ -412,8 +394,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
             )
 
             try:
-                row = self._get_metrics(
-                    metric_type.value,
+                row = self._get_metric_fn[metric_type.value](
                     metrics,
                     runner=runner,
                     session=session,
@@ -471,7 +452,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                         profile_results["columns"][column].update(
                             {
                                 "name": column,
-                                "timestamp": datetime.now(tz=timezone.utc).timestamp(),
+                                "timestamp": int(
+                                    datetime.now(tz=timezone.utc).timestamp() * 1000
+                                ),
                                 **profile,
                             }
                         )

metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py

@@ -0,0 +1,86 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Interfaces with database for all database engine
+supporting sqlalchemy abstraction layer
+"""
+
+from typing import List
+
+from sqlalchemy.exc import ProgrammingError
+
+from metadata.profiler.interface.sqlalchemy.profiler_interface import (
+    SQAProfilerInterface,
+    handle_query_exception,
+)
+from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.processor.runner import QueryRunner
+from metadata.profiler.source.single_store.metrics.window.first_quartile import (
+    SingleStoreFirstQuartile,
+)
+from metadata.profiler.source.single_store.metrics.window.median import (
+    SingleStoreMedian,
+)
+from metadata.profiler.source.single_store.metrics.window.third_quartile import (
+    SingleStoreThirdQuartile,
+)
+from metadata.utils.logger import profiler_interface_registry_logger
+
+logger = profiler_interface_registry_logger()
+
+
+class SingleStoreProfilerInterface(SQAProfilerInterface):
+    """
+    Interface to interact with registry supporting
+    sqlalchemy.
+    """
+
+    def _compute_window_metrics(
+        self,
+        metrics: List[Metrics],
+        runner: QueryRunner,
+        *args,
+        **kwargs,
+    ):
+        """Given a list of metrics, compute the given results
+        and returns the values
+
+        Args:
+            column: the column to compute the metrics against
+            metrics: list of metrics to compute
+        Returns:
+            dictionnary of results
+        """
+        session = kwargs.get("session")
+        column = kwargs.get("column")
+
+        if not metrics:
+            return None
+        try:
+            # we patch the metrics at runtime to use the SingleStore specific functions
+            # as we can't compile the query based on the dialect as it return `mysql`
+            metrics = [SingleStoreFirstQuartile, SingleStoreMedian, SingleStoreThirdQuartile]  # type: ignore
+            row = runner.select_first_from_sample(
+                *[metric(column).fn() for metric in metrics],
+            )
+        except ProgrammingError:
+            logger.info(
+                f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to overflow"
+            )
+            return None
+
+        except Exception as exc:
+            msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
+            handle_query_exception(msg, exc, session)
+        if row:
+            return dict(row)
+        return None

metadata/profiler/metrics/system/queries/redshift.py

@@ -35,19 +35,17 @@ STL_QUERY = """
         sti."database",
         sti."schema",
         sti."table",
-        sq.text,
         DATE_TRUNC('second', data.starttime) AS starttime
     FROM
         data
         INNER JOIN  pg_catalog.svv_table_info sti ON data.tbl = sti.table_id
-        INNER JOIN pg_catalog.stl_querytext sq ON data.query = sq.query
     where
         sti."database" = '{database}' AND
        	sti."schema" = '{schema}' AND
         "rows" != 0 AND
         DATE(data.starttime) >= CURRENT_DATE - 1
-    GROUP BY 2,3,4,5,6
-    ORDER BY 6 DESC
+    GROUP BY 2,3,4,5
+    ORDER BY 5 DESC
 """
 
 
@@ -73,7 +71,7 @@ def get_query_results(
             database_name=row.database,
             schema_name=row.schema,
             table_name=row.table,
-            query_text=row.text,
+            query_text=None,
             query_type=operation,
             timestamp=row.starttime,
             rows=row.rows,

metadata/profiler/metrics/system/system.py

@@ -113,10 +113,6 @@ def _(
     dataset_id = table.__table_args__["schema"]  # type: ignore
 
     metric_results: List[Dict] = []
-    # QueryResult = namedtuple(
-    #     "QueryResult",
-    #     "query_type,timestamp,destination_table,dml_statistics",
-    # )
 
     jobs = get_value_from_cache(
         SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.BigQuery}.{project_id}.{dataset_id}.jobs"

metadata/profiler/metrics/window/first_quartile.py

@@ -19,15 +19,15 @@ from typing import List, cast
 from sqlalchemy import column
 
 from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.functions.median import MedianFn
 from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
 from metadata.utils.logger import profiler_logger
 
 logger = profiler_logger()
 
 
-class FirstQuartile(StaticMetric):
+class FirstQuartile(StaticMetric, PercentilMixin):
     """
     First Quartile Metric
 
@@ -53,14 +53,14 @@ class FirstQuartile(StaticMetric):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
             # col fullname is only needed for MySQL and SQLite
-            return MedianFn(
+            return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.25,
             )
 
         if is_concatenable(self.col.type):
-            return MedianFn(
+            return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.25,

metadata/profiler/metrics/window/median.py

@@ -19,15 +19,15 @@ from typing import List, cast
 from sqlalchemy import column
 
 from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.functions.median import MedianFn
 from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
 from metadata.utils.logger import profiler_logger
 
 logger = profiler_logger()
 
 
-class Median(StaticMetric):
+class Median(StaticMetric, PercentilMixin):
     """
     Median Metric
 
@@ -53,14 +53,14 @@ class Median(StaticMetric):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
             # col fullname is only needed for MySQL and SQLite
-            return MedianFn(
+            return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.5,
             )
 
         if is_concatenable(self.col.type):
-            return MedianFn(
+            return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.5,

metadata/profiler/metrics/window/percentille_mixin.py

@@ -0,0 +1,9 @@
+"""function calls shared accross all percentile metrics"""
+
+from metadata.profiler.orm.functions.median import MedianFn
+
+
+class PercentilMixin:
+    def _compute_sqa_fn(self, column, table, percentile):
+        """Generic method to compute the quartile using sqlalchemy"""
+        return MedianFn(column, table, percentile)

metadata/profiler/metrics/window/third_quartile.py

@@ -19,15 +19,15 @@ from typing import List, cast
 from sqlalchemy import column
 
 from metadata.profiler.metrics.core import StaticMetric, _label
+from metadata.profiler.metrics.window.percentille_mixin import PercentilMixin
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.functions.median import MedianFn
 from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
 from metadata.utils.logger import profiler_logger
 
 logger = profiler_logger()
 
 
-class ThirdQuartile(StaticMetric):
+class ThirdQuartile(StaticMetric, PercentilMixin):
     """
     Third Quartile Metric
 
@@ -53,14 +53,14 @@ class ThirdQuartile(StaticMetric):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
             # col fullname is only needed for MySQL and SQLite
-            return MedianFn(
+            return self._compute_sqa_fn(
                 column(self.col.name, self.col.type),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.75,
             )
 
         if is_concatenable(self.col.type):
-            return MedianFn(
+            return self._compute_sqa_fn(
                 LenFn(column(self.col.name, self.col.type)),
                 self.col.table.fullname if self.col.table is not None else None,
                 0.75,

metadata/profiler/processor/core.py

@@ -92,7 +92,7 @@ class Profiler(Generic[TMetric]):
         self.include_columns = include_columns
         self.exclude_columns = exclude_columns
         self._metrics = metrics
-        self._profile_date = datetime.now(tz=timezone.utc).timestamp()
+        self._profile_date = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
         self.profile_sample_config = self.profiler_interface.profile_sample_config
 
         self.validate_composed_metric()

metadata/profiler/source/single_store/functions/median.py

@@ -0,0 +1,17 @@
+"""Median function for single store"""
+
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.functions import FunctionElement
+
+from metadata.profiler.metrics.core import CACHE
+
+
+class SingleStoreMedianFn(FunctionElement):
+    inherit_cache = CACHE
+
+
+@compiles(SingleStoreMedianFn)
+def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
+    col = compiler.process(elements.clauses.clauses[0])
+    percentile = elements.clauses.clauses[2].value
+    return f"approx_percentile({col}, {percentile:.2f})"

metadata/profiler/source/single_store/metrics/window/first_quartile.py

@@ -0,0 +1,10 @@
+"""Override first quartile metric definition for SingleStore"""
+
+from metadata.profiler.metrics.window.first_quartile import FirstQuartile
+from metadata.profiler.source.single_store.functions.median import SingleStoreMedianFn
+
+
+class SingleStoreFirstQuartile(FirstQuartile):
+    def _compute_sqa_fn(self, column, table, percentile):
+        """Generic method to compute the quartile using sqlalchemy"""
+        return SingleStoreMedianFn(column, table, percentile)

metadata/profiler/source/single_store/metrics/window/median.py

@@ -0,0 +1,10 @@
+"""Override first quartile metric definition for SingleStore"""
+
+from metadata.profiler.metrics.window.median import Median
+from metadata.profiler.source.single_store.functions.median import SingleStoreMedianFn
+
+
+class SingleStoreMedian(Median):
+    def _compute_sqa_fn(self, column, table, percentile):
+        """Generic method to compute the quartile using sqlalchemy"""
+        return SingleStoreMedianFn(column, table, percentile)

metadata/profiler/source/single_store/metrics/window/third_quartile.py

@@ -0,0 +1,10 @@
+"""Override first quartile metric definition for SingleStore"""
+
+from metadata.profiler.metrics.window.third_quartile import ThirdQuartile
+from metadata.profiler.source.single_store.functions.median import SingleStoreMedianFn
+
+
+class SingleStoreThirdQuartile(ThirdQuartile):
+    def _compute_sqa_fn(self, column, table, percentile):
+        """Generic method to compute the quartile using sqlalchemy"""
+        return SingleStoreMedianFn(column, table, percentile)