openmetadata-ingestion 1.5.0.0rc2__py3-none-any.whl → 1.5.2.0__py3-none-any.whl

metadata/ingestion/source/database/saphana/lineage.py

@@ -0,0 +1,159 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+import traceback
+from typing import Iterable, Optional
+
+from sqlalchemy import text
+
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.connections.database.sapHanaConnection import (
+    SapHanaConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.database.saphana.cdata_parser import (
+    ParsedLineage,
+    parse_registry,
+)
+from metadata.ingestion.source.database.saphana.models import SapHanaLineageModel
+from metadata.ingestion.source.database.saphana.queries import SAPHANA_LINEAGE
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import get_ssl_connection
+
+logger = ingestion_logger()
+
+
+class SaphanaLineageSource(Source):
+    """
+    Get the lineage information of:
+    - calculationview
+    - analyticview
+    - attributeview
+
+    We support the following relationships:
+    - Analytic View and Attribute View based on a Table
+    - Calculation View based on an Analytic View, Attribute View, Calculation View or Table
+
+    Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT
+    """
+
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+        get_engine: bool = True,
+    ):
+        super().__init__()
+        self.config = config
+        self.metadata = metadata
+        self.service_connection = self.config.serviceConnection.root.config
+        self.source_config = self.config.sourceConfig.config
+        self.engine = (
+            get_ssl_connection(self.service_connection) if get_engine else None
+        )
+
+        logger.info(
+            "Initializing SAP Hana Lineage Source. Note that we'll parse the lineage from CDATA XML definition "
+            + "from _SYS_REPO.ACTIVE_OBJECT and we won't use the time-specific input parameters."
+        )
+
+    def prepare(self):
+        """By default, there's nothing to prepare"""
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: SapHanaConnection = config.serviceConnection.root.config
+        if not isinstance(connection, SapHanaConnection):
+            raise InvalidSourceException(
+                f"Expected SapHanaConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def close(self) -> None:
+        self.engine.dispose()
+
+    def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Based on the query logs, prepare the lineage
+        and send it to the sink
+        """
+        with self.engine.connect() as conn:
+            result = conn.execution_options(
+                stream_results=True, max_row_buffer=100
+            ).execute(text(SAPHANA_LINEAGE))
+            for row in result:
+                try:
+                    lineage_model = SapHanaLineageModel.validate(dict(row))
+
+                    yield from self.parse_cdata(
+                        metadata=self.metadata, lineage_model=lineage_model
+                    )
+                except Exception as exc:
+                    self.status.failed(
+                        error=StackTraceError(
+                            name=row["OBJECT_NAME"],
+                            error=f"Error validating lineage model due to [{exc}]",
+                            stackTrace=traceback.format_exc(),
+                        )
+                    )
+
+    def parse_cdata(
+        self, metadata: OpenMetadata, lineage_model: SapHanaLineageModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT"""
+        parse_fn = parse_registry.registry.get(lineage_model.object_suffix.value)
+        try:
+            parsed_lineage: ParsedLineage = parse_fn(lineage_model.cdata)
+            to_entity: Table = metadata.get_by_name(
+                entity=Table,
+                fqn=lineage_model.get_fqn(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                ),
+            )
+
+            if to_entity:
+                yield from parsed_lineage.to_request(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                    to_entity=to_entity,
+                )
+        except Exception as exc:
+            error = (
+                f"Error parsing CDATA XML for {lineage_model.object_suffix} at "
+                + f"{lineage_model.package_id}/{lineage_model.object_name} due to [{exc}]"
+            )
+            self.status.failed(
+                error=StackTraceError(
+                    name=lineage_model.object_name,
+                    error=error,
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def test_connection(self) -> None:
+        test_connection_fn = get_test_connection_fn(self.service_connection)
+        test_connection_fn(self.engine)

metadata/ingestion/source/database/saphana/models.py

@@ -0,0 +1,66 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+from enum import Enum
+
+from pydantic import Field, computed_field
+from typing_extensions import Annotated
+
+from metadata.generated.schema.entity.data.table import Table
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils import fqn
+
+SYS_BIC_SCHEMA_NAME = "_SYS_BIC"
+
+
+class ViewType(Enum):
+    """Supported SAP Hana Views"""
+
+    CALCULATION_VIEW = "calculationview"
+    ANALYTIC_VIEW = "analyticview"
+    ATTRIBUTE_VIEW = "attributeview"
+    # Artificially set to define calculationView internal models. This won't come from the ACTIVE_OBJECT table
+    LOGICAL = "logical"
+    DATA_BASE_TABLE = "table"
+
+
+class SapHanaLineageModel(BaseModel):
+    """SAP Hana Lineage model from _SYS_REPO.ACTIVE_OBJECT"""
+
+    package_id: Annotated[
+        str, Field(..., description="Package ID that hosts the model code")
+    ]
+    object_name: Annotated[str, Field(..., description="View Name")]
+    object_suffix: Annotated[ViewType, Field(..., description="View Type")]
+    cdata: Annotated[str, Field(..., description="XML representation of the model")]
+
+    @computed_field
+    @property
+    def name(self) -> str:
+        """
+        Actual name representation of the view in _SYS_BIC.VIEWS.
+        This is the name in OpenMetadata
+        """
+        return f"{self.package_id}/{self.object_name}"
+
+    def get_fqn(self, metadata: OpenMetadata, service_name: str) -> str:
+        """Build OM's FQN with the static schema name from _SYS_BIC"""
+        return fqn.build(
+            metadata,
+            entity_type=Table,
+            service_name=service_name,
+            database_name=None,
+            schema_name=SYS_BIC_SCHEMA_NAME,
+            table_name=self.name,
+        )

metadata/ingestion/source/database/saphana/queries.py

@@ -0,0 +1,23 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana DB queries for metadata extraction
+"""
+
+SAPHANA_LINEAGE = """
+SELECT
+  PACKAGE_ID,
+  OBJECT_NAME,
+  OBJECT_SUFFIX,
+  CDATA
+FROM _SYS_REPO.ACTIVE_OBJECT
+WHERE OBJECT_SUFFIX IN ('analyticview', 'attributeview', 'calculationview');
+"""

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -21,7 +21,7 @@ import threading
 import traceback
 from collections import defaultdict
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from sqlalchemy import Column, inspect, text
 from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
@@ -451,14 +451,16 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     column=metric_func.column,
                     sample=sample,
                 )
-                if row:
-                    for k, v in row.items():
-                        # Replace NaN values with None
-                        if isinstance(v, float) and math.isnan(v):
-                            logger.warning(
-                                "NaN data detected and will be cast to null in OpenMetadata to maintain database parity"
-                            )
-                            row[k] = None
+                if row and isinstance(row, dict):
+                    row = self._validate_nulls(row)
+
+                # System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
+                if row and metric_func.metric_type == MetricTypes.System:
+                    row = [
+                        self._validate_nulls(r) if isinstance(r, dict) else r
+                        for r in row
+                    ]
+
             except Exception as exc:
                 error = (
                     f"{metric_func.column if metric_func.column is not None else metric_func.table.__tablename__} "
@@ -476,6 +478,17 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
 
             return row, column, metric_func.metric_type.value
 
+    @staticmethod
+    def _validate_nulls(row: Dict[str, Any]) -> Dict[str, Any]:
+        """Detect if we are computing NaNs and replace them with None"""
+        for k, v in row.items():
+            if isinstance(v, float) and math.isnan(v):
+                logger.warning(
+                    "NaN data detected and will be cast to null in OpenMetadata to maintain database parity"
+                )
+                row[k] = None
+        return row
+
     # pylint: disable=use-dict-literal
     def get_all_metrics(
         self,

metadata/profiler/metrics/static/mean.py

@@ -15,7 +15,7 @@ AVG Metric definition
 from functools import partial
 from typing import Callable, List, Optional, cast
 
-from sqlalchemy import column, func
+from sqlalchemy import column
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.sql.functions import GenericFunction
 
@@ -39,20 +39,19 @@ from metadata.utils.logger import profiler_logger
 logger = profiler_logger()
 
 
-# pylint: disable=invalid-name
-class avg(GenericFunction):
+class AvgFn(GenericFunction):
     name = "avg"
     inherit_cache = CACHE
 
 
-@compiles(avg, Dialects.ClickHouse)
+@compiles(AvgFn, Dialects.ClickHouse)
 def _(element, compiler, **kw):
     """Handle case for empty table. If empty, clickhouse returns NaN"""
     proc = compiler.process(element.clauses, **kw)
     return f"if(isNaN(avg({proc})), null, avg({proc}))"
 
 
-@compiles(avg, Dialects.Redshift)
+@compiles(AvgFn, Dialects.Redshift)
 def _(element, compiler, **kw):
     """
     Cast to decimal to get around potential integer overflow error
@@ -61,7 +60,7 @@ def _(element, compiler, **kw):
     return f"avg(CAST({proc} AS DECIMAL(38,0)))"
 
 
-@compiles(avg, Dialects.MSSQL)
+@compiles(AvgFn, Dialects.MSSQL)
 def _(element, compiler, **kw):
     """
     Cast to decimal to get around potential integer overflow error -
@@ -71,7 +70,7 @@ def _(element, compiler, **kw):
     return f"avg(cast({proc} as decimal))"
 
 
-@compiles(avg, Dialects.Trino)
+@compiles(AvgFn, Dialects.Trino)
 def _(element, compiler, **kw):
     proc = compiler.process(element.clauses, **kw)
     first_clause = element.clauses.clauses[0]
@@ -108,10 +107,10 @@ class Mean(StaticMetric):
     def fn(self):
         """sqlalchemy function"""
         if is_quantifiable(self.col.type):
-            return func.avg(column(self.col.name, self.col.type))
+            return AvgFn(column(self.col.name, self.col.type))
 
         if is_concatenable(self.col.type):
-            return func.avg(LenFn(column(self.col.name, self.col.type)))
+            return AvgFn(LenFn(column(self.col.name, self.col.type)))
 
         logger.debug(
             f"Don't know how to process type {self.col.type} when computing MEAN"

metadata/profiler/orm/functions/median.py

@@ -28,12 +28,19 @@ logger = profiler_logger()
 class MedianFn(FunctionElement):
     inherit_cache = CACHE
 
+    @staticmethod
+    def default_fn(elements, compiler, **kwargs):  # pylint: disable=unused-argument
+        col = compiler.process(elements.clauses.clauses[0])
+        percentile = elements.clauses.clauses[2].value
+        return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (
+            percentile,
+            col,
+        )
+
 
 @compiles(MedianFn)
-def _(elements, compiler, **kwargs):  # pylint: disable=unused-argument
-    col = compiler.process(elements.clauses.clauses[0])
-    percentile = elements.clauses.clauses[2].value
-    return "percentile_cont(%.2f) WITHIN GROUP (ORDER BY %s ASC)" % (percentile, col)
+def _(elements, compiler, **kwargs):
+    return MedianFn.default_fn(elements, compiler, **kwargs)
 
 
 @compiles(MedianFn, Dialects.BigQuery)

metadata/profiler/orm/registry.py

@@ -162,7 +162,9 @@ def is_quantifiable(_type) -> bool:
     """
     if isinstance(_type, DataType):
         return _type.value in QUANTIFIABLE_SET
-    return is_numeric(_type) or is_integer(_type)
+    return (
+        is_numeric(_type) or is_integer(_type) or getattr(_type, "quantifiable", False)
+    )
 
 
 def is_concatenable(_type) -> bool:

metadata/utils/importer.py

@@ -12,6 +12,7 @@
 Helpers to import python classes and modules dynamically
 """
 import importlib
+import sys
 import traceback
 from enum import Enum
 from typing import Any, Callable, Optional, Type, TypeVar
@@ -27,6 +28,7 @@ from metadata.generated.schema.metadataIngestion.workflow import Sink as Workflo
 from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage
 from metadata.utils.class_helper import get_service_type_from_source_type
 from metadata.utils.logger import utils_logger
+from metadata.utils.singleton import Singleton
 
 logger = utils_logger()
 
@@ -244,3 +246,27 @@ def import_test_case_class(
             test_definition_class,
         )
     )
+
+
+class SideEffectsLoader(metaclass=Singleton):
+    modules = set(sys.modules.keys())
+
+    def import_side_effects(self, *modules):
+        """Handles loading of side effects and caches modules that have already been imported.
+        Requires full module name."""
+        for module in modules:
+            if module not in self.modules:
+                try:
+                    module = importlib.import_module(module)
+                    SideEffectsLoader.modules.add(module.__name__)
+                except Exception as err:
+                    logger.debug(traceback.format_exc())
+                    raise DynamicImportException(
+                        f"Cannot load object from {module} due to {err}"
+                    )
+            else:
+                logger.debug(f"Module {module} already imported")
+
+
+def import_side_effects(*modules):
+    SideEffectsLoader().import_side_effects(*modules)