openmetadata-ingestion 1.5.1.0__py3-none-any.whl → 1.5.2.0__py3-none-any.whl

metadata/ingestion/source/database/saphana/lineage.py

@@ -0,0 +1,159 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+import traceback
+from typing import Iterable, Optional
+
+from sqlalchemy import text
+
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.connections.database.sapHanaConnection import (
+    SapHanaConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.connections import get_test_connection_fn
+from metadata.ingestion.source.database.saphana.cdata_parser import (
+    ParsedLineage,
+    parse_registry,
+)
+from metadata.ingestion.source.database.saphana.models import SapHanaLineageModel
+from metadata.ingestion.source.database.saphana.queries import SAPHANA_LINEAGE
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import get_ssl_connection
+
+logger = ingestion_logger()
+
+
+class SaphanaLineageSource(Source):
+    """
+    Get the lineage information of:
+    - calculationview
+    - analyticview
+    - attributeview
+
+    We support the following relationships:
+    - Analytic View and Attribute View based on a Table
+    - Calculation View based on an Analytic View, Attribute View, Calculation View or Table
+
+    Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT
+    """
+
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata: OpenMetadata,
+        get_engine: bool = True,
+    ):
+        super().__init__()
+        self.config = config
+        self.metadata = metadata
+        self.service_connection = self.config.serviceConnection.root.config
+        self.source_config = self.config.sourceConfig.config
+        self.engine = (
+            get_ssl_connection(self.service_connection) if get_engine else None
+        )
+
+        logger.info(
+            "Initializing SAP Hana Lineage Source. Note that we'll parse the lineage from CDATA XML definition "
+            + "from _SYS_REPO.ACTIVE_OBJECT and we won't use the time-specific input parameters."
+        )
+
+    def prepare(self):
+        """By default, there's nothing to prepare"""
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.model_validate(config_dict)
+        connection: SapHanaConnection = config.serviceConnection.root.config
+        if not isinstance(connection, SapHanaConnection):
+            raise InvalidSourceException(
+                f"Expected SapHanaConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def close(self) -> None:
+        self.engine.dispose()
+
+    def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Based on the query logs, prepare the lineage
+        and send it to the sink
+        """
+        with self.engine.connect() as conn:
+            result = conn.execution_options(
+                stream_results=True, max_row_buffer=100
+            ).execute(text(SAPHANA_LINEAGE))
+            for row in result:
+                try:
+                    lineage_model = SapHanaLineageModel.validate(dict(row))
+
+                    yield from self.parse_cdata(
+                        metadata=self.metadata, lineage_model=lineage_model
+                    )
+                except Exception as exc:
+                    self.status.failed(
+                        error=StackTraceError(
+                            name=row["OBJECT_NAME"],
+                            error=f"Error validating lineage model due to [{exc}]",
+                            stackTrace=traceback.format_exc(),
+                        )
+                    )
+
+    def parse_cdata(
+        self, metadata: OpenMetadata, lineage_model: SapHanaLineageModel
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """Parse the CDATA XML definition from _SYS_REPO.ACTIVE_OBJECT"""
+        parse_fn = parse_registry.registry.get(lineage_model.object_suffix.value)
+        try:
+            parsed_lineage: ParsedLineage = parse_fn(lineage_model.cdata)
+            to_entity: Table = metadata.get_by_name(
+                entity=Table,
+                fqn=lineage_model.get_fqn(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                ),
+            )
+
+            if to_entity:
+                yield from parsed_lineage.to_request(
+                    metadata=metadata,
+                    service_name=self.config.serviceName,
+                    to_entity=to_entity,
+                )
+        except Exception as exc:
+            error = (
+                f"Error parsing CDATA XML for {lineage_model.object_suffix} at "
+                + f"{lineage_model.package_id}/{lineage_model.object_name} due to [{exc}]"
+            )
+            self.status.failed(
+                error=StackTraceError(
+                    name=lineage_model.object_name,
+                    error=error,
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def test_connection(self) -> None:
+        test_connection_fn = get_test_connection_fn(self.service_connection)
+        test_connection_fn(self.engine)

metadata/ingestion/source/database/saphana/models.py

@@ -0,0 +1,66 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana lineage module
+"""
+from enum import Enum
+
+from pydantic import Field, computed_field
+from typing_extensions import Annotated
+
+from metadata.generated.schema.entity.data.table import Table
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils import fqn
+
+SYS_BIC_SCHEMA_NAME = "_SYS_BIC"
+
+
+class ViewType(Enum):
+    """Supported SAP Hana Views"""
+
+    CALCULATION_VIEW = "calculationview"
+    ANALYTIC_VIEW = "analyticview"
+    ATTRIBUTE_VIEW = "attributeview"
+    # Artificially set to define calculationView internal models. This won't come from the ACTIVE_OBJECT table
+    LOGICAL = "logical"
+    DATA_BASE_TABLE = "table"
+
+
+class SapHanaLineageModel(BaseModel):
+    """SAP Hana Lineage model from _SYS_REPO.ACTIVE_OBJECT"""
+
+    package_id: Annotated[
+        str, Field(..., description="Package ID that hosts the model code")
+    ]
+    object_name: Annotated[str, Field(..., description="View Name")]
+    object_suffix: Annotated[ViewType, Field(..., description="View Type")]
+    cdata: Annotated[str, Field(..., description="XML representation of the model")]
+
+    @computed_field
+    @property
+    def name(self) -> str:
+        """
+        Actual name representation of the view in _SYS_BIC.VIEWS.
+        This is the name in OpenMetadata
+        """
+        return f"{self.package_id}/{self.object_name}"
+
+    def get_fqn(self, metadata: OpenMetadata, service_name: str) -> str:
+        """Build OM's FQN with the static schema name from _SYS_BIC"""
+        return fqn.build(
+            metadata,
+            entity_type=Table,
+            service_name=service_name,
+            database_name=None,
+            schema_name=SYS_BIC_SCHEMA_NAME,
+            table_name=self.name,
+        )

metadata/ingestion/source/database/saphana/queries.py

@@ -0,0 +1,23 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SAP Hana DB queries for metadata extraction
+"""
+
+SAPHANA_LINEAGE = """
+SELECT
+  PACKAGE_ID,
+  OBJECT_NAME,
+  OBJECT_SUFFIX,
+  CDATA
+FROM _SYS_REPO.ACTIVE_OBJECT
+WHERE OBJECT_SUFFIX IN ('analyticview', 'attributeview', 'calculationview');
+"""