openmetadata-ingestion 1.5.0.0rc1__py3-none-any.whl → 1.5.1.0__py3-none-any.whl

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -16,11 +16,12 @@ supporting sqlalchemy abstraction layer
 """
 
 import concurrent.futures
+import math
 import threading
 import traceback
 from collections import defaultdict
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from sqlalchemy import Column, inspect, text
 from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
@@ -450,6 +451,16 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     column=metric_func.column,
                     sample=sample,
                 )
+                if row and isinstance(row, dict):
+                    row = self._validate_nulls(row)
+
+                # System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
+                if row and metric_func.metric_type == MetricTypes.System:
+                    row = [
+                        self._validate_nulls(r) if isinstance(r, dict) else r
+                        for r in row
+                    ]
+
             except Exception as exc:
                 error = (
                     f"{metric_func.column if metric_func.column is not None else metric_func.table.__tablename__} "
@@ -467,6 +478,17 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
 
             return row, column, metric_func.metric_type.value
 
+    @staticmethod
+    def _validate_nulls(row: Dict[str, Any]) -> Dict[str, Any]:
+        """Detect if we are computing NaNs and replace them with None"""
+        for k, v in row.items():
+            if isinstance(v, float) and math.isnan(v):
+                logger.warning(
+                    "NaN data detected and will be cast to null in OpenMetadata to maintain database parity"
+                )
+                row[k] = None
+        return row
+
     # pylint: disable=use-dict-literal
     def get_all_metrics(
         self,

metadata/profiler/metrics/composed/null_ratio.py

@@ -49,12 +49,9 @@ class NullRatio(ComposedMetric):
         Safely compute null ratio based on the profiler
         results of other Metrics
         """
-        import pandas as pd
 
-        res_count = res.get(Count.name())
-        res_null = res.get(NullCount.name())
-
-        if not pd.isnull(res_count) and not pd.isnull(res_null):
-            result = res_null / (res_null + res_count)
-            return None if pd.isnull(result) else result
-        return None
+        count = res.get(Count.name())
+        null_count = res.get(NullCount.name())
+        if count + null_count == 0:
+            return None
+        return null_count / (null_count + count)

metadata/profiler/metrics/hybrid/histogram.py

@@ -25,7 +25,11 @@ from metadata.profiler.metrics.static.count import Count
 from metadata.profiler.metrics.static.max import Max
 from metadata.profiler.metrics.static.min import Min
 from metadata.profiler.orm.functions.length import LenFn
-from metadata.profiler.orm.registry import is_concatenable, is_quantifiable
+from metadata.profiler.orm.registry import (
+    is_concatenable,
+    is_quantifiable,
+    is_value_non_numeric,
+)
 from metadata.utils.helpers import format_large_string_numbers
 from metadata.utils.logger import profiler_logger
 
@@ -120,7 +124,6 @@ class Histogram(HybridMetric):
             # freedman-diaconis rule
             bin_width = self._get_bin_width(float(res_iqr), res_row_count)  # type: ignore
             num_bins = math.ceil((res_max - res_min) / bin_width)  # type: ignore
-
         # sturge's rule
         if res_iqr is None or num_bins > max_bin_count:
             num_bins = int(math.ceil(math.log2(res_row_count) + 1))
@@ -148,7 +151,10 @@ class Histogram(HybridMetric):
                 "We are missing the session attribute to compute the Histogram."
             )
 
-        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)):
+        if not (is_quantifiable(self.col.type) or is_concatenable(self.col.type)) or (
+            is_value_non_numeric(res.get(Min.name()))
+            or is_value_non_numeric(res.get(Max.name()))
+        ):
             return None
 
         # get the metric need for the freedman-diaconis rule

metadata/profiler/metrics/static/mean.py

@@ -15,7 +15,7 @@ AVG Metric definition
 from functools import partial
 from typing import Callable, List, Optional, cast
 
-from sqlalchemy import column, func
+from sqlalchemy import column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
@@ -39,20 +39,19 @@ from metadata.utils.logger import profiler_logger
 logger = profiler_logger()
 
 
-# pylint: disable=invalid-name
-class avg(GenericFunction):
+class AvgFn(GenericFunction):
     name = "avg"
     inherit_cache = CACHE
 
 
-@compiles(avg, Dialects.ClickHouse)
+@compiles(AvgFn, Dialects.ClickHouse)
 def _(element, compiler, **kw):
     """Handle case for empty table. If empty, clickhouse returns NaN"""
     proc = compiler.process(element.clauses, **kw)
     return f"if(isNaN(avg({proc})), null, avg({proc}))"
 
 
-@compiles(avg, Dialects.Redshift)
+@compiles(AvgFn, Dialects.Redshift)
 def _(element, compiler, **kw):
     """
     Cast to decimal to get around potential integer overflow error
@@ -61,7 +60,7 @@ def _(element, compiler, **kw):
     return f"avg(CAST({proc} AS DECIMAL(38,0)))"
 
 
-@compiles(avg, Dialects.MSSQL)
+@compiles(AvgFn, Dialects.MSSQL)
 def _(element, compiler, **kw):
     """
     Cast to decimal to get around potential integer overflow error -
@@ -71,7 +70,7 @@ def _(element, compiler, **kw):
     return f"avg(cast({proc} as decimal))"
 
 
-@compiles(avg, Dialects.Trino)
+@compiles(AvgFn, Dialects.Trino)
 def _(element, compiler, **kw):
     proc = compiler.process(element.clauses, **kw)
     first_clause = element.clauses.clauses[0]
@@ -108,10 +107,10 @@ class Mean(StaticMetric):
     def fn(self):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
-            return func.avg(column(self.col.name, self.col.type))
+            return AvgFn(column(self.col.name, self.col.type))
 
         if is_concatenable(self.col.type):
-            return func.avg(LenFn(column(self.col.name, self.col.type)))
+            return AvgFn(LenFn(column(self.col.name, self.col.type)))
 
         logger.debug(
             f"Don't know how to process type {self.col.type} when computing MEAN"

metadata/profiler/metrics/static/null_count.py

@@ -16,6 +16,7 @@ Null Count Metric definition
 
 
 from sqlalchemy import case, column
+from sqlalchemy.sql.functions import coalesce
 
 from metadata.generated.schema.configuration.profilerConfiguration import MetricType
 from metadata.profiler.metrics.core import StaticMetric, _label
@@ -48,8 +49,9 @@ class NullCount(StaticMetric):
     @_label
     def fn(self):
         """sqlalchemy function"""
-        return SumFn(
-            case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)
+        return coalesce(
+            SumFn(case([(column(self.col.name, self.col.type).is_(None), 1)], else_=0)),
+            0,
         )
 
     def df_fn(self, dfs=None):

metadata/profiler/orm/functions/median.py

@@ -28,12 +28,19 @@ logger = profiler_logger()
 class MedianFn(FunctionElement):
     inherit_cache = CACHE
 
+    @staticmethod
+    def default_fn(elements, compiler, **kwargs):  # pylint: disable=unused-argument
+        col = compiler.process(elements.clauses.clauses[0])
+        percentile = elements.clauses.clauses[2].value
+        return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (
+            percentile,
+            col,
+        )
+
 
 @compiles(MedianFn)
-def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
-    col = compiler.process(elements.clauses.clauses[0])
-    percentile = elements.clauses.clauses[2].value
-    return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (percentile, col)
+def _(elements, compiler, **kwargs):
+    return MedianFn.default_fn(elements, compiler, **kwargs)
 
 
 @compiles(MedianFn, Dialects.BigQuery)

metadata/profiler/orm/functions/sum.py

@@ -45,7 +45,7 @@ def _(element, compiler, **kw):
 def _(element, compiler, **kw):
     """Cast to DECIMAL to address cannot cast nan to bigint"""
     proc = compiler.process(element.clauses, **kw)
-    return f"SUM(TRY_CAST({proc} AS DECIMAL))"
+    return f"COALESCE(SUM(CAST({proc} AS DECIMAL)),0)"
 
 
 @compiles(SumFn, Dialects.BigQuery)

metadata/profiler/orm/functions/table_metric_computer.py

@@ -194,7 +194,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("object_name").label("table_name"),
                     Column("created"),
                 ],
-                self._build_table("all_objects", None),
+                self._build_table("DBA_OBJECTS", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("object_name")) == self.table_name.lower(),
@@ -209,7 +209,7 @@ class OracleTableMetricComputer(BaseTableMetricComputer):
                     Column("table_name"),
                     Column("NUM_ROWS"),
                 ],
-                self._build_table("all_tables", None),
+                self._build_table("DBA_TABLES", None),
                 [
                     func.lower(Column("owner")) == self.schema_name.lower(),
                     func.lower(Column("table_name")) == self.table_name.lower(),

metadata/profiler/orm/registry.py

@@ -13,6 +13,8 @@
 Custom types' registry for easy access
 without having an import mess
 """
+import math
+
 import sqlalchemy
 from sqlalchemy import Date, DateTime, Integer, Numeric, Time
 from sqlalchemy.sql.sqltypes import Concatenable, Enum
@@ -160,7 +162,9 @@ def is_quantifiable(_type) -> bool:
     """
     if isinstance(_type, DataType):
         return _type.value in QUANTIFIABLE_SET
-    return is_numeric(_type) or is_integer(_type)
+    return (
+        is_numeric(_type) or is_integer(_type) or getattr(_type, "quantifiable", False)
+    )
 
 
 def is_concatenable(_type) -> bool:
@@ -171,3 +175,12 @@ def is_concatenable(_type) -> bool:
     if isinstance(_type, DataType):
         return _type.value in CONCATENABLE_SET
     return issubclass(_type.__class__, Concatenable)
+
+
+def is_value_non_numeric(value) -> bool:
+    try:
+        if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
+            return True
+        return False
+    except Exception:
+        return False

metadata/profiler/processor/default.py

@@ -77,9 +77,12 @@ class DefaultProfiler(Profiler):
         include_columns: Optional[List[ColumnProfilerConfig]] = None,
         exclude_columns: Optional[List[str]] = None,
         global_profiler_configuration: Optional[ProfilerConfiguration] = None,
+        db_service=None,
     ):
         _metrics = get_default_metrics(
-            table=profiler_interface.table, ometa_client=profiler_interface.ometa_client
+            table=profiler_interface.table,
+            ometa_client=profiler_interface.ometa_client,
+            db_service=db_service,
         )
 
         super().__init__(

metadata/profiler/processor/sampler/sampler_factory.py

@@ -15,6 +15,9 @@ Factory class for creating sampler objects
 
 from typing import Union
 
+from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
+    AzureSQLConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
@@ -36,6 +39,9 @@ from metadata.generated.schema.entity.services.connections.database.trinoConnect
 from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
 from metadata.profiler.processor.sampler.nosql.sampler import NoSQLSampler
 from metadata.profiler.processor.sampler.pandas.sampler import DatalakeSampler
+from metadata.profiler.processor.sampler.sqlalchemy.azuresql.sampler import (
+    AzureSQLSampler,
+)
 from metadata.profiler.processor.sampler.sqlalchemy.bigquery.sampler import (
     BigQuerySampler,
 )
@@ -89,3 +95,6 @@ sampler_factory_.register(
 sampler_factory_.register(
     source_type=DynamoDBConnection.__name__, sampler_class=NoSQLSampler
 )
+sampler_factory_.register(
+    source_type=AzureSQLConnection.__name__, sampler_class=AzureSQLSampler
+)

metadata/profiler/processor/sampler/sqlalchemy/azuresql/sampler.py

@@ -0,0 +1,40 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Helper module to handle data sampling
+for the profiler
+"""
+from typing import List, Optional
+
+from sqlalchemy import Column
+
+from metadata.generated.schema.entity.data.table import TableData
+from metadata.profiler.processor.sampler.sqlalchemy.sampler import SQASampler
+
+
+class AzureSQLSampler(SQASampler):
+    """
+    Generates a sample of the data to not
+    run the query in the whole table.
+    """
+
+    # These types are not supported by pyodbc - it throws
+    # an error when trying to fetch data from these columns
+    # pyodbc.ProgrammingError: ('ODBC SQL type -151 is not yet supported.  column-index=x  type=-151', 'HY106')
+    NOT_COMPUTE_PYODBC = {"SQASGeography", "UndeterminedType"}
+
+    def fetch_sample_data(self, columns: Optional[List[Column]] = None) -> TableData:
+        sqa_columns = []
+        if columns:
+            for col in columns:
+                if col.type.__class__.__name__ not in self.NOT_COMPUTE_PYODBC:
+                    sqa_columns.append(col)
+        return super().fetch_sample_data(sqa_columns or columns)

metadata/profiler/source/base/profiler_source.py

@@ -277,6 +277,7 @@ class ProfilerSource(ProfilerSourceInterface):
                 include_columns=self._get_include_columns(entity, table_config),
                 exclude_columns=self._get_exclude_columns(entity, table_config),
                 global_profiler_configuration=self.global_profiler_configuration,
+                db_service=db_service,
             )
 
         metrics = (

metadata/readers/file/api_reader.py

@@ -21,12 +21,15 @@ from metadata.generated.schema.security.credentials.bitbucketCredentials import
 from metadata.generated.schema.security.credentials.githubCredentials import (
     GitHubCredentials,
 )
+from metadata.generated.schema.security.credentials.gitlabCredentials import (
+    GitlabCredentials,
+)
 from metadata.readers.file.base import Reader
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
-ReadersCredentials = Union[GitHubCredentials, BitBucketCredentials]
+ReadersCredentials = Union[GitHubCredentials, BitBucketCredentials, GitlabCredentials]
 
 
 class ApiReader(Reader, ABC):

metadata/readers/file/gitlab.py

@@ -0,0 +1,172 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Gitlab client to read files with token auth
+"""
+import base64
+import traceback
+from enum import Enum
+from typing import Any, Dict, List, Optional
+from urllib.parse import quote_plus
+
+import requests
+
+from metadata.generated.schema.security.credentials.gitlabCredentials import (
+    GitlabCredentials,
+)
+from metadata.readers.file.api_reader import ApiReader
+from metadata.readers.file.base import ReadException
+from metadata.utils.constants import UTF_8
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+HOST = "https://gitlab.com/api/v4"
+
+
+class UrlParts(Enum):
+    FILES = "files"
+    PROJECTS = "projects"
+    REPOSITORY = "repository"
+    TREE = "tree"
+
+
+class GitlabReader(ApiReader):
+    """
+    Handle calls to the Gitlab API against a repo
+    """
+
+    credentials: GitlabCredentials
+
+    def __init__(self, credentials):
+        super().__init__(credentials)
+        self._encoded_project_path = None
+
+    @property
+    def auth_headers(self) -> Dict[str, str]:
+        """
+        Build the headers to authenticate
+        to the API
+        """
+        if self._auth_headers is None and self.credentials.token:
+            self._auth_headers = {
+                "PRIVATE-TOKEN": self.credentials.token.root.get_secret_value()
+            }
+
+        return self._auth_headers
+
+    @property
+    def encoded_project_path(self) -> str:
+        """
+        Build the URL-encoded project path for the Gitlab API
+        """
+        if (
+            self._encoded_project_path is None
+            and self.credentials.repositoryOwner.root
+            and self.credentials.repositoryName.root
+        ):
+            self._encoded_project_path = quote_plus(
+                "/".join(
+                    [
+                        self.credentials.repositoryOwner.root,
+                        self.credentials.repositoryName.root,
+                    ]
+                )
+            )
+
+        return self._encoded_project_path
+
+    @staticmethod
+    def _decode_content(json_response: Dict[str, Any]) -> str:
+        """
+        Return the content of the response
+
+        If no `content` there, throw the KeyError
+        """
+        return base64.b64decode(json_response["content"]).decode(UTF_8)
+
+    def read(self, path: str, **__) -> str:
+        """
+        Read a file from a Gitlab Repo and return its
+        contents as a string
+        https://docs.gitlab.com/ee/api/repository_files.html
+        """
+        encoded_file_path = quote_plus(path)
+        branch = self._get_default_branch()
+        try:
+            res = requests.get(
+                self._build_url(
+                    HOST,
+                    UrlParts.PROJECTS.value,
+                    self.encoded_project_path,
+                    UrlParts.REPOSITORY.value,
+                    UrlParts.FILES.value,
+                    f"{encoded_file_path}?ref={branch}",
+                ),
+                headers=self.auth_headers,
+                timeout=30,
+            )
+            if res.status_code == 200:
+                return self._decode_content(res.json())
+
+            # If we don't get a 200, raise
+            res.raise_for_status()
+
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            raise ReadException(f"Error fetching file [{path}] from repo: {err}")
+
+        raise ReadException(f"Could not fetch file [{path}] from repo")
+
+    def _get_default_branch(self) -> str:
+        """
+        Get repo default branch
+        """
+        res = requests.get(
+            self._build_url(HOST, UrlParts.PROJECTS.value, self.encoded_project_path),
+            headers=self.auth_headers,
+            timeout=30,
+        )
+        if res.status_code == 200:
+            return res.json().get("default_branch")
+
+        # If we don't get a 200, raise
+        res.raise_for_status()
+        raise RuntimeError("Could not fetch the default branch")
+
+    def _get_tree(self, url: str = None) -> Optional[List[str]]:
+        """
+        Use the Gitlab Repository Tree API to iterate over tree pages recursively
+        """
+        if url is None:
+            url = self._build_url(
+                HOST,
+                UrlParts.PROJECTS.value,
+                self.encoded_project_path,
+                UrlParts.REPOSITORY.value,
+                f"{UrlParts.TREE.value}?recursive=true&pagination=keyset&per_page=100&order_by=path&sort=desc",
+            )
+        res = requests.get(
+            url,
+            headers=self.auth_headers,
+            timeout=30,
+        )
+        if res.status_code == 200:
+            paths = [elem.get("path") for elem in res.json()]
+            if res.links.get("next"):
+                paths.extend(self._get_tree(res.links["next"]["url"]))
+            return paths
+
+        # If we don't get a 200, raise
+        res.raise_for_status()
+
+        return None

metadata/utils/helpers.py

@@ -106,8 +106,11 @@ def pretty_print_time_duration(duration: Union[int, float]) -> str:
     """
 
     days = divmod(duration, 86400)[0]
+    duration = duration - days * 86400
     hours = divmod(duration, 3600)[0]
+    duration = duration - hours * 3600
     minutes = divmod(duration, 60)[0]
+    duration = duration - minutes * 60
     seconds = round(divmod(duration, 60)[1], 2)
     if days:
         return f"{days}day(s) {hours}h {minutes}m {seconds}s"
@@ -123,7 +126,7 @@ def get_start_and_end(duration: int = 0) -> Tuple[datetime, datetime]:
     Method to return start and end time based on duration
     """
 
-    today = datetime.now(timezone.utc)
+    today = datetime.now(timezone.utc).replace(tzinfo=None)
     start = (today + timedelta(0 - duration)).replace(
         hour=0, minute=0, second=0, microsecond=0
     )

metadata/utils/importer.py

@@ -12,6 +12,7 @@
 Helpers to import python classes and modules dynamically
 """
 import importlib
+import sys
 import traceback
 from enum import Enum
 from typing import Any, Callable, Optional, Type, TypeVar
@@ -27,6 +28,7 @@ from metadata.generated.schema.metadataIngestion.workflow import Sink as Workflo
 from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage
 from metadata.utils.class_helper import get_service_type_from_source_type
 from metadata.utils.logger import utils_logger
+from metadata.utils.singleton import Singleton
 
 logger = utils_logger()
 
@@ -244,3 +246,27 @@ def import_test_case_class(
             test_definition_class,
         )
     )
+
+
+class SideEffectsLoader(metaclass=Singleton):
+    modules = set(sys.modules.keys())
+
+    def import_side_effects(self, *modules):
+        """Handles loading of side effects and caches modules that have already been imported.
+        Requires full module name."""
+        for module in modules:
+            if module not in self.modules:
+                try:
+                    module = importlib.import_module(module)
+                    SideEffectsLoader.modules.add(module.__name__)
+                except Exception as err:
+                    logger.debug(traceback.format_exc())
+                    raise DynamicImportException(
+                        f"Cannot load object from {module} due to {err}"
+                    )
+            else:
+                logger.debug(f"Module {module} already imported")
+
+
+def import_side_effects(*modules):
+    SideEffectsLoader().import_side_effects(*modules)