openmetadata-ingestion 1.5.1.0__py3-none-any.whl → 1.5.3.0__py3-none-any.whl

metadata/ingestion/source/database/saphana/lineage.py

@@ -0,0 +1,159 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+import traceback
+from typing import Iterable, Optional
+
+from sqlalchemy import text
+
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.connections.database.sapHanaConnection import (
+    SapHanaConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.database.saphana.cdata_parser import (
+    ParsedLineage,
+    parse_registry,
+)
+from metadata.ingestion.source.database.saphana.models import SapHanaLineageModel
+from metadata.ingestion.source.database.saphana.queries import SAPHANA_LINEAGE
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import get_ssl_connection
+
+logger = ingestion_logger()
+
+
+class SaphanaLineageSource(Source):
+    """
+    Get the lineage information of:
+    - calculationview
+    - analyticview
+    - attributeview
+
+    We support the following relationships:
+    - Analytic View and Attribute View based on a Table
+    - Calculation View based on an Analytic View, Attribute View, Calculation View or Table
+
+    Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT
+    """
+
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+        get_engine: bool = True,
+    ):
+        super().__init__()
+        self.config = config
+        self.metadata = metadata
+        self.service_connection = self.config.serviceConnection.root.config
+        self.source_config = self.config.sourceConfig.config
+        self.engine = (
+            get_ssl_connection(self.service_connection) if get_engine else None
+        )
+
+        logger.info(
+            "Initializing SAP Hana Lineage Source. Note that we'll parse the lineage from CDATA XML definition "
+            + "from _SYS_REPO.ACTIVE_OBJECT and we won't use the time-specific input parameters."
+        )
+
+    def prepare(self):
+        """By default, there's nothing to prepare"""
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: SapHanaConnection = config.serviceConnection.root.config
+        if not isinstance(connection, SapHanaConnection):
+            raise InvalidSourceException(
+                f"Expected SapHanaConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def close(self) -> None:
+        self.engine.dispose()
+
+    def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Based on the query logs, prepare the lineage
+        and send it to the sink
+        """
+        with self.engine.connect() as conn:
+            result = conn.execution_options(
+                stream_results=True, max_row_buffer=100
+            ).execute(text(SAPHANA_LINEAGE))
+            for row in result:
+                try:
+                    lineage_model = SapHanaLineageModel.validate(dict(row))
+
+                    yield from self.parse_cdata(
+                        metadata=self.metadata, lineage_model=lineage_model
+                    )
+                except Exception as exc:
+                    self.status.failed(
+                        error=StackTraceError(
+                            name=row["OBJECT_NAME"],
+                            error=f"Error validating lineage model due to [{exc}]",
+                            stackTrace=traceback.format_exc(),
+                        )
+                    )
+
+    def parse_cdata(
+        self, metadata: OpenMetadata, lineage_model: SapHanaLineageModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT"""
+        parse_fn = parse_registry.registry.get(lineage_model.object_suffix.value)
+        try:
+            parsed_lineage: ParsedLineage = parse_fn(lineage_model.cdata)
+            to_entity: Table = metadata.get_by_name(
+                entity=Table,
+                fqn=lineage_model.get_fqn(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                ),
+            )
+
+            if to_entity:
+                yield from parsed_lineage.to_request(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                    to_entity=to_entity,
+                )
+        except Exception as exc:
+            error = (
+                f"Error parsing CDATA XML for {lineage_model.object_suffix} at "
+                + f"{lineage_model.package_id}/{lineage_model.object_name} due to [{exc}]"
+            )
+            self.status.failed(
+                error=StackTraceError(
+                    name=lineage_model.object_name,
+                    error=error,
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def test_connection(self) -> None:
+        test_connection_fn = get_test_connection_fn(self.service_connection)
+        test_connection_fn(self.engine)

metadata/ingestion/source/database/saphana/models.py

@@ -0,0 +1,66 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+from enum import Enum
+
+from pydantic import Field, computed_field
+from typing_extensions import Annotated
+
+from metadata.generated.schema.entity.data.table import Table
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils import fqn
+
+SYS_BIC_SCHEMA_NAME = "_SYS_BIC"
+
+
+class ViewType(Enum):
+    """Supported SAP Hana Views"""
+
+    CALCULATION_VIEW = "calculationview"
+    ANALYTIC_VIEW = "analyticview"
+    ATTRIBUTE_VIEW = "attributeview"
+    # Artificially set to define calculationView internal models. This won't come from the ACTIVE_OBJECT table
+    LOGICAL = "logical"
+    DATA_BASE_TABLE = "table"
+
+
+class SapHanaLineageModel(BaseModel):
+    """SAP Hana Lineage model from _SYS_REPO.ACTIVE_OBJECT"""
+
+    package_id: Annotated[
+        str, Field(..., description="Package ID that hosts the model code")
+    ]
+    object_name: Annotated[str, Field(..., description="View Name")]
+    object_suffix: Annotated[ViewType, Field(..., description="View Type")]
+    cdata: Annotated[str, Field(..., description="XML representation of the model")]
+
+    @computed_field
+    @property
+    def name(self) -> str:
+        """
+        Actual name representation of the view in _SYS_BIC.VIEWS.
+        This is the name in OpenMetadata
+        """
+        return f"{self.package_id}/{self.object_name}"
+
+    def get_fqn(self, metadata: OpenMetadata, service_name: str) -> str:
+        """Build OM's FQN with the static schema name from _SYS_BIC"""
+        return fqn.build(
+            metadata,
+            entity_type=Table,
+            service_name=service_name,
+            database_name=None,
+            schema_name=SYS_BIC_SCHEMA_NAME,
+            table_name=self.name,
+        )

metadata/ingestion/source/database/saphana/queries.py

@@ -0,0 +1,23 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana DB queries for metadata extraction
+"""
+
+SAPHANA_LINEAGE = """
+SELECT
+  PACKAGE_ID,
+  OBJECT_NAME,
+  OBJECT_SUFFIX,
+  CDATA
+FROM _SYS_REPO.ACTIVE_OBJECT
+WHERE OBJECT_SUFFIX IN ('analyticview', 'attributeview', 'calculationview');
+"""

metadata/pii/processor.py

@@ -66,6 +66,7 @@ class PIIProcessor(Processor):
         )  # Used to satisfy type checked
 
         self._ner_scanner = None
+        self.name_scanner = ColumnNameScanner()
         self.confidence_threshold = self.source_config.confidence
 
     @property
@@ -128,7 +129,7 @@ class PIIProcessor(Processor):
             return None
 
         # Scan by column name. If no results there, check the sample data, if any
-        tag_and_confidence = ColumnNameScanner.scan(column.name.root) or (
+        tag_and_confidence = self.name_scanner.scan(column.name.root) or (
             self.ner_scanner.scan([row[idx] for row in table_data.rows])
             if table_data
             else None

metadata/pii/scanners/base.py

@@ -0,0 +1,23 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Basic Scanner ABC
+"""
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseScanner(ABC):
+    """Basic scanner abstract class"""
+
+    @abstractmethod
+    def scan(self, data: Any):
+        """Scan the given data from a column"""

metadata/pii/scanners/column_name_scanner.py

@@ -17,17 +17,18 @@ from typing import Optional
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII
 from metadata.pii.models import TagAndConfidence, TagType
+from metadata.pii.scanners.base import BaseScanner
 from metadata.utils import fqn
 
 
-class ColumnNameScanner:
+class ColumnNameScanner(BaseScanner):
     """
     Column Name Scanner to scan column name
     """
 
     sensitive_regex = {
         "PASSWORD": re.compile("^.*password.*$", re.IGNORECASE),
-        "SSN": re.compile("^.*(ssn|social).*$", re.IGNORECASE),
+        "US_SSN": re.compile("^.*(ssn|social).*$", re.IGNORECASE),
         "CREDIT_CARD": re.compile("^.*(credit).*(card).*$", re.IGNORECASE),
         "BANK_ACCOUNT": re.compile("^.*bank.*(acc|num).*$", re.IGNORECASE),
         "EMAIL_ADDRESS": re.compile("^.*(email|e-mail|mail).*$", re.IGNORECASE),
@@ -53,14 +54,13 @@ class ColumnNameScanner:
         "PHONE_NUMBER": re.compile("^.*(phone).*$", re.IGNORECASE),
     }
 
-    @classmethod
-    def scan(cls, column_name: str) -> Optional[TagAndConfidence]:
+    def scan(self, data: str) -> Optional[TagAndConfidence]:
         """
         Check the column name against the regex patterns and prepare the
         sensitive or non-sensitive tag
         """
-        for pii_type_pattern in cls.sensitive_regex.values():
-            if pii_type_pattern.match(column_name) is not None:
+        for pii_type_pattern in self.sensitive_regex.values():
+            if pii_type_pattern.match(data) is not None:
                 return TagAndConfidence(
                     tag_fqn=fqn.build(
                         metadata=None,
@@ -71,8 +71,8 @@ class ColumnNameScanner:
                     confidence=1,
                 )
 
-        for pii_type_pattern in cls.non_sensitive_regex.values():
-            if pii_type_pattern.match(column_name) is not None:
+        for pii_type_pattern in self.non_sensitive_regex.values():
+            if pii_type_pattern.match(data) is not None:
                 return TagAndConfidence(
                     tag_fqn=fqn.build(
                         metadata=None,

metadata/pii/scanners/ner_scanner.py

@@ -13,9 +13,10 @@ NER Scanner based on Presidio.
 
 Supported Entities https://microsoft.github.io/presidio/supported_entities/
 """
+import json
 import traceback
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from pydantic import BaseModel
 
@@ -23,6 +24,7 @@ from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.pii.constants import PII, SPACY_EN_MODEL
 from metadata.pii.models import TagAndConfidence
 from metadata.pii.ner import NEREntity
+from metadata.pii.scanners.base import BaseScanner
 from metadata.utils import fqn
 from metadata.utils.logger import pii_logger
 
@@ -39,7 +41,7 @@ class StringAnalysis(BaseModel):
 
 
 # pylint: disable=import-outside-toplevel
-class NERScanner:
+class NERScanner(BaseScanner):
     """
     Based on https://microsoft.github.io/presidio/
     """
@@ -74,7 +76,7 @@ class NERScanner:
         )
         return top_entity, entities_score[top_entity].score
 
-    def scan(self, sample_data_rows: List[Any]) -> Optional[TagAndConfidence]:
+    def scan(self, data: List[Any]) -> Optional[TagAndConfidence]:
         """
         Scan the column's sample data rows and look for PII.
 
@@ -95,24 +97,17 @@ class NERScanner:
            be thought as the "score" times "weighted down appearances".
         4. Once we have the "top" `Entity` from that column, we assign the PII label accordingly from `NEREntity`.
         """
-        logger.debug("Processing '%s'", sample_data_rows)
+        logger.debug("Processing '%s'", data)
 
         # Initialize an empty dict for the given row list
         entities_score: Dict[str, StringAnalysis] = defaultdict(
             lambda: StringAnalysis(score=0, appearances=0)
         )
 
-        str_sample_data_rows = [str(row) for row in sample_data_rows if row is not None]
+        str_sample_data_rows = [str(row) for row in data if row is not None]
         for row in str_sample_data_rows:
             try:
-                results = self.analyzer.analyze(row, language="en")
-                for result in results:
-                    entities_score[result.entity_type] = StringAnalysis(
-                        score=result.score
-                        if result.score > entities_score[result.entity_type].score
-                        else entities_score[result.entity_type].score,
-                        appearances=entities_score[result.entity_type].appearances + 1,
-                    )
+                self.process_data(row=row, entities_score=entities_score)
             except Exception as exc:
                 logger.warning(f"Unknown error while processing {row} - {exc}")
                 logger.debug(traceback.format_exc())
@@ -133,3 +128,38 @@ class NERScanner:
             )
 
         return None
+
+    def process_data(self, row: str, entities_score: Dict[str, StringAnalysis]) -> None:
+        """Process the Sample Data rows, checking if they are of JSON format as well"""
+        # first, check if the data is JSON or we can work with strings
+        is_json, value = self.is_json_data(row)
+        if is_json and isinstance(value, dict):
+            for val in value.values():
+                self.process_data(row=str(val), entities_score=entities_score)
+        elif is_json and isinstance(value, list):
+            for val in value:
+                self.process_data(row=str(val), entities_score=entities_score)
+        else:
+            self.scan_value(value=row, entities_score=entities_score)
+
+    @staticmethod
+    def is_json_data(value: str) -> Tuple[bool, Union[dict, list, None]]:
+        """Check if the value is a JSON object that we need to process differently than strings"""
+        try:
+            res = json.loads(value)
+            if isinstance(res, (dict, list)):
+                return True, res
+            return False, None
+        except json.JSONDecodeError:
+            return False, None
+
+    def scan_value(self, value: str, entities_score: Dict[str, StringAnalysis]):
+        """Scan the value for PII"""
+        results = self.analyzer.analyze(value, language="en")
+        for result in results:
+            entities_score[result.entity_type] = StringAnalysis(
+                score=result.score
+                if result.score > entities_score[result.entity_type].score
+                else entities_score[result.entity_type].score,
+                appearances=entities_score[result.entity_type].appearances + 1,
+            )

metadata/profiler/source/metadata.py

@@ -12,7 +12,7 @@
 OpenMetadata source for the profiler
 """
 import traceback
-from typing import Iterable, Optional, cast
+from typing import Iterable, List, Optional, cast
 
 from pydantic import BaseModel, ConfigDict
 
@@ -265,6 +265,14 @@ class OpenMetadataSource(Source):
 
         return databases
 
+    def _get_fields(self) -> List[str]:
+        """Get the fields required to process the tables"""
+        return (
+            TABLE_FIELDS
+            if not self.source_config.processPiiSensitive
+            else TABLE_FIELDS + TAGS_FIELD
+        )
+
     def get_table_entities(self, database):
         """
         List and filter OpenMetadata tables based on the
@@ -282,9 +290,7 @@ class OpenMetadataSource(Source):
         """
         tables = self.metadata.list_all_entities(
             entity=Table,
-            fields=TABLE_FIELDS
-            if not self.source_config.processPiiSensitive
-            else TABLE_FIELDS + TAGS_FIELD,
+            fields=self._get_fields(),
             params={
                 "service": self.config.source.serviceName,
                 "database": fqn.build(

metadata/profiler/source/metadata_ext.py

@@ -26,7 +26,6 @@ from typing import Iterable, cast
 from sqlalchemy.inspection import inspect
 
 from metadata.generated.schema.entity.data.database import Database
-from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -138,7 +137,7 @@ class OpenMetadataSourceExt(OpenMetadataSource):
                             service_name=None,
                             schema_name=schema_name,
                             table_name=table_name,
-                            fields="tableProfilerConfig",
+                            fields=",".join(self._get_fields()),
                         )
                         if not table_entity:
                             logger.debug(
@@ -226,36 +225,3 @@ class OpenMetadataSourceExt(OpenMetadataSource):
         except Exception as exc:
             logger.debug(f"Failed to fetch database names {exc}")
             logger.debug(traceback.format_exc())
-
-    def get_table_entities(self, database):
-        """
-        List and filter OpenMetadata tables based on the
-        source configuration.
-
-        The listing will be based on the entities from the
-        informed service name in the source configuration.
-
-        Note that users can specify `table_filter_pattern` to
-        either be `includes` or `excludes`. This means
-        that we will either what is specified in `includes`
-        or we will use everything but the tables excluded.
-
-        Same with `schema_filter_pattern`.
-        """
-        tables = self.metadata.list_all_entities(
-            entity=Table,
-            fields=[
-                "tableProfilerConfig",
-            ],
-            params={
-                "service": self.config.source.serviceName,
-                "database": fqn.build(
-                    self.metadata,
-                    entity_type=Database,
-                    service_name=self.config.source.serviceName,
-                    database_name=database.name.root,
-                ),
-            },  # type: ignore
-        )
-
-        yield from self.filter_entities(tables)

metadata/workflow/profiler.py

@@ -46,8 +46,8 @@ class ProfilerWorkflow(IngestionWorkflow):
         if self.config.source.serviceName:
             return OpenMetadataSource
         logger.info(
-            "Database Service name not provided, we will scan all the tables"
-            "available within data source and locate table entity in OpenMetadata"
+            "Database Service name not provided, we will scan all the tables "
+            "available within data source and locate table entity in OpenMetadata "
             "to ingest profiler data."
         )
         return OpenMetadataSourceExt