openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/database/iceberg/helper.py

@@ -0,0 +1,124 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Iceberg source helpers.
+"""
+from __future__ import annotations
+
+from itertools import takewhile
+from typing import Optional, Tuple
+
+import pyiceberg.partitioning
+import pyiceberg.table
+import pyiceberg.types
+
+from metadata.generated.schema.entity.data.table import Column, Constraint, DataType
+
+
+def namespace_to_str(namespace: tuple[str]) -> str:
+    """Turns a PyIceberg Namespace into a String.
+
+    The PyIceberg namespaces are returned as tuples and we turn them into a String
+    concatenating the items with a '.' in between.
+    """
+    return ".".join(namespace)
+
+
+def get_table_name_as_str(table: pyiceberg.table.Table) -> str:
+    """Returns the Table Name as Tring from a PyIceberg Table.
+
+    The PyIceberg table name is returned as tuple and we turn them into a String
+    concatenating the items with a '.' in between.
+    """
+    # We are skipping the first item because it is the schema name.
+    return ".".join(table.name()[1:])
+
+
+def get_column_from_partition(
+    columns: Tuple[pyiceberg.types.NestedField, ...],
+    partition: pyiceberg.partitioning.PartitionField,
+) -> str:
+    """Returns the Column Name belonging to a partition."""
+    # A Partition in Iceberg has a Source Column to which a Transformation is applied.
+    # We need to return the Source Column name.
+    return [
+        column.name for column in columns if column.field_id == partition.source_id
+    ][0]
+
+
+def get_owner_from_table(
+    table: pyiceberg.table.Table, property_key: str
+) -> Optional[str]:
+    """Retrives the owner information from given Table Property."""
+    return table.properties.get(property_key)
+
+
+class IcebergColumnParser:
+    """Responsible for containing the logic to parse a column from PyIceberg to OpenMetadata"""
+
+    data_type_map = {
+        # Primitive
+        "BOOLEAN": DataType.BOOLEAN,
+        "INT": DataType.INT,
+        "LONG": DataType.LONG,
+        "FLOAT": DataType.FLOAT,
+        "DOUBLE": DataType.DOUBLE,
+        "DECIMAL": DataType.DECIMAL,
+        "DATE": DataType.DATE,
+        "TIME": DataType.TIME,
+        "TIMESTAMP": DataType.TIMESTAMP,
+        "TIMESTAMPTZ": DataType.TIMESTAMPZ,
+        "STRING": DataType.STRING,
+        "UUID": DataType.UUID,
+        "FIXED": DataType.FIXED,
+        "BINARY": DataType.BINARY,
+        # Nested
+        "STRUCT": DataType.STRUCT,
+        "LIST": DataType.ARRAY,
+        "MAP": DataType.MAP,
+    }
+
+    @classmethod
+    def parse(cls, field: pyiceberg.types.NestedField) -> Column:
+        """Parses a PyIceberg Field into an OpenMetadata Column."""
+        data_type = cls.data_type_map.get(
+            "".join(takewhile(lambda x: x.isalpha(), str(field.field_type))).upper(),
+            DataType.UNKNOWN,
+        )
+
+        column_def = {
+            "name": field.name,
+            "description": field.doc,
+            "dataType": data_type,
+            "dataTypeDisplay": str(field.field_type),
+            "constraint": Constraint.NOT_NULL if field.required else None,
+        }
+
+        if data_type == DataType.ARRAY:
+            array_data_type = cls.data_type_map.get(
+                "".join(
+                    takewhile(lambda x: x.isalpha(), str(field.field_type.element_type))
+                ).upper(),
+                DataType.UNKNOWN,
+            )
+            column_def["arrayDataType"] = array_data_type
+        # Iceberg Binary type has no length. Using a noop 0 value.
+        elif data_type == DataType.BINARY:
+            column_def["dataLength"] = 0
+        elif data_type == DataType.DECIMAL:
+            column_def["precision"] = field.field_type.precision
+            column_def["scale"] = field.field_type.scale
+        elif data_type == DataType.STRUCT:
+            column_def["children"] = [
+                cls.parse(child) for child in field.field_type.fields
+            ]
+
+        return Column(**column_def)

metadata/ingestion/source/database/iceberg/metadata.py

@@ -0,0 +1,311 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Iceberg source methods.
+"""
+import traceback
+from typing import Any, Iterable, Optional, Tuple, Union
+
+import pyiceberg
+import pyiceberg.exceptions
+
+from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
+from metadata.generated.schema.api.data.createDatabaseSchema import (
+    CreateDatabaseSchemaRequest,
+)
+from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
+from metadata.generated.schema.api.data.createStoredProcedure import (
+    CreateStoredProcedureRequest,
+)
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
+from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.table import Table, TableType
+from metadata.generated.schema.entity.services.connections.database.icebergConnection import (
+    IcebergConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
+    DatabaseServiceMetadataPipeline,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.connections import get_connection
+from metadata.ingestion.source.database.database_service import DatabaseServiceSource
+from metadata.ingestion.source.database.iceberg.helper import (
+    get_owner_from_table,
+    get_table_name_as_str,
+    namespace_to_str,
+)
+from metadata.ingestion.source.database.iceberg.models import IcebergTable
+from metadata.utils import fqn
+from metadata.utils.filters import filter_by_schema, filter_by_table
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class IcebergSource(DatabaseServiceSource):
+    """
+    Implements the necessary methods to extract Iceberg metadata.
+    Relies on [PyIceberg](https://py.iceberg.apache.org/).
+    """
+
+    def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
+        super().__init__()
+        self.config = config
+        self.source_config: DatabaseServiceMetadataPipeline = (
+            self.config.sourceConfig.config
+        )
+        self.metadata = metadata
+        self.service_connection = self.config.serviceConnection.__root__.config
+        self.iceberg = get_connection(self.service_connection)
+
+        self.connection_obj = self.iceberg
+        self.test_connection()
+
+    @classmethod
+    def create(cls, config_dict, metadata: OpenMetadata):
+        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
+        connection: IcebergConnection = config.serviceConnection.__root__.config
+        if not isinstance(connection, IcebergConnection):
+            raise InvalidSourceException(
+                f"Expected GlueConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def get_database_names(self) -> Iterable[str]:
+        """
+        Prepares the database name to be sent to stage.
+        Filtering happens here.
+        """
+        yield self.service_connection.catalog.databaseName or "default"
+
+    def yield_database(
+        self, database_name: str
+    ) -> Iterable[Either[CreateDatabaseRequest]]:
+        """
+        From topology.
+        Prepare a database request and pass it to the sink.
+
+        Also, update the self.inspector value to the current db.
+        """
+        yield Either(
+            right=CreateDatabaseRequest(
+                name=database_name,
+                service=self.context.database_service,
+            )
+        )
+
+    def get_database_schema_names(self) -> Iterable[str]:
+        """
+        Prepares the database schema name to be sent to stage.
+        Filtering happens here.
+        """
+        for namespace in self.iceberg.list_namespaces():
+            namespace_name = namespace_to_str(namespace)
+            try:
+                schema_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.database_service,
+                    database_name=self.context.database,
+                    schema_name=namespace_name,
+                )
+                if filter_by_schema(
+                    self.config.sourceConfig.config.schemaFilterPattern,
+                    schema_fqn
+                    if self.config.sourceConfig.config.useFqnForFiltering
+                    else namespace_name,
+                ):
+                    self.status.filter(schema_fqn, "Schema Filtered Out")
+                    continue
+                yield namespace_name
+            except Exception as exc:
+                self.status.failed(
+                    StackTraceError(
+                        name=namespace_name,
+                        error=f"Unexpected exception to get the namespace [{namespace_name}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def yield_database_schema(
+        self, schema_name: str
+    ) -> Iterable[Either[CreateDatabaseSchemaRequest]]:
+        """
+        From topology.
+        Prepare a database request and pass it to the sink.
+        """
+        yield Either(
+            right=CreateDatabaseSchemaRequest(
+                name=schema_name,
+                database=fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.database_service,
+                    database_name=self.context.database,
+                ),
+            )
+        )
+
+    def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
+        """
+        Prepares the table name to be sent to stage.
+        Filtering happens here.
+        """
+        namespace = self.context.database_schema
+
+        for table_identifier in self.iceberg.list_tables(namespace):
+            try:
+                table = self.iceberg.load_table(table_identifier)
+                table_name = get_table_name_as_str(table)
+                table_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=Table,
+                    service_name=self.context.database_service,
+                    database_name=self.context.database,
+                    schema_name=self.context.database_schema,
+                    table_name=table_name,
+                )
+                if filter_by_table(
+                    self.config.sourceConfig.config.tableFilterPattern,
+                    table_fqn
+                    if self.config.sourceConfig.config.useFqnForFiltering
+                    else table_name,
+                ):
+                    self.status.filter(
+                        table_fqn,
+                        "Table Filtered Out",
+                    )
+                    continue
+
+                self.context.iceberg_table = table
+                yield table_name, TableType.Regular
+            except pyiceberg.exceptions.NoSuchPropertyException:
+                logger.warning(
+                    f"Table [{table_identifier}] does not have the 'table_type' property. Skipped."
+                )
+                continue
+            except pyiceberg.exceptions.NoSuchIcebergTableError:
+                logger.warning(
+                    f"Table [{table_identifier}] is not an Iceberg Table. Skipped."
+                )
+                continue
+            except pyiceberg.exceptions.NoSuchTableError:
+                logger.warning(f"Table [{table_identifier}] not Found. Skipped.")
+                continue
+            except Exception as exc:
+                table_name = ".".join(table_identifier)
+                self.status.failed(
+                    StackTraceError(
+                        name=table_name,
+                        error=f"Unexpected exception to get table [{table_name}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def get_owner_ref(self, table_name: str) -> Optional[EntityReference]:
+        owner = get_owner_from_table(
+            self.context.iceberg_table, self.service_connection.ownershipProperty
+        )
+        try:
+            if owner:
+                owner_reference = self.metadata.get_reference_by_email(owner)
+                return owner_reference
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
+        return None
+
+    def yield_table(
+        self, table_name_and_type: Tuple[str, TableType]
+    ) -> Iterable[Either[CreateTableRequest]]:
+        """
+        From topology.
+        Prepare a table request and pass it to the sink.
+
+        Also, update the self.inspector value to the current db.
+        """
+        table_name, table_type = table_name_and_type
+        iceberg_table = self.context.iceberg_table
+        try:
+            owner = self.get_owner_ref(table_name)
+            table = IcebergTable.from_pyiceberg(
+                table_name, table_type, owner, iceberg_table
+            )
+            table_request = CreateTableRequest(
+                name=table.name,
+                tableType=table.tableType,
+                description=table.description,
+                owner=table.owner,
+                columns=table.columns,
+                tablePartition=table.tablePartition,
+                databaseSchema=fqn.build(
+                    metadata=self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.database_service,
+                    database_name=self.context.database,
+                    schema_name=self.context.database_schema,
+                ),
+            )
+            yield Either(right=table_request)
+            self.register_record(table_request=table_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=table_name,
+                    error=f"Unexpected exception to yield table [{table_name}]: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def yield_tag(
+        self, schema_name: str
+    ) -> Iterable[Either[OMetaTagAndClassification]]:
+        """
+        From topology. To be run for each schema
+        """
+        yield from []
+
+    def yield_view_lineage(self) -> Iterable[Either[AddLineageRequest]]:
+        """
+        From topology.
+        Parses view definition to get lineage information
+        """
+        yield from []
+
+    def get_stored_procedures(self) -> Iterable[Any]:
+        """Not Implemented"""
+
+    def yield_stored_procedure(
+        self, stored_procedure: Any
+    ) -> Iterable[Either[CreateStoredProcedureRequest]]:
+        """Process the stored procedure information"""
+        yield from []
+
+    def yield_procedure_lineage_and_queries(
+        self,
+    ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
+        """Extracts the lineage information from Stored Procedures"""
+        yield from []
+
+    def close(self):
+        """There is no connection to close."""

metadata/ingestion/source/database/iceberg/models.py

@@ -0,0 +1,66 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Iceberg source models.
+"""
+from __future__ import annotations
+
+from typing import List, Optional
+
+import pyiceberg.table
+from pydantic import BaseModel
+
+from metadata.generated.schema.entity.data.table import (
+    Column,
+    TablePartition,
+    TableType,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.source.database.iceberg.helper import (
+    IcebergColumnParser,
+    get_column_from_partition,
+)
+
+
+class IcebergTable(BaseModel):
+    name: str
+    tableType: TableType
+    description: Optional[str]
+    owner: Optional[EntityReference]
+    columns: List[Column] = []
+    tablePartition: Optional[TablePartition]
+
+    @classmethod
+    def from_pyiceberg(
+        cls,
+        name: str,
+        table_type: TableType,
+        owner: Optional[EntityReference],
+        table: pyiceberg.table.Table,
+    ) -> IcebergTable:
+        """Responsible for parsing the needed information from a PyIceberg Table."""
+        iceberg_columns = table.schema().fields
+
+        return IcebergTable(
+            name=name,
+            tableType=table_type,
+            description=table.properties.get("comment"),
+            owner=owner,
+            columns=[IcebergColumnParser.parse(column) for column in iceberg_columns],
+            tablePartition=TablePartition(
+                columns=[
+                    get_column_from_partition(iceberg_columns, partition)
+                    for partition in table.spec().fields
+                ],
+                intervalType=None,
+                interval=None,
+            ),
+        )

metadata/ingestion/source/database/life_cycle_query_mixin.py

@@ -15,19 +15,27 @@ import traceback
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional
 
 from pydantic import BaseModel, Field
 from sqlalchemy.engine import Engine
 
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
     DatabaseServiceMetadataPipeline,
 )
-from metadata.ingestion.api.models import StackTraceError
+from metadata.generated.schema.type.lifeCycle import AccessDetails, LifeCycle
+from metadata.ingestion.api.models import Either, Entity
 from metadata.ingestion.api.status import Status
+from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.topology import TopologyContext
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.time_utils import convert_timestamp_to_milliseconds
 
 logger = ingestion_logger()
 
@@ -78,8 +86,69 @@ class LifeCycleQueryMixin:
                     StackTraceError(
                         name="Life Cycle Queries",
                         error=f"Error trying to get life cycle information due to [{exc}]",
-                        stack_trace=traceback.format_exc(),
+                        stackTrace=traceback.format_exc(),
                     )
                 )
 
         return queries_dict
+
+    def get_life_cycle_data(self, entity: Entity, query: str):
+        """
+        Get the life cycle data
+        """
+        if entity:
+            try:
+                life_cycle_data = self.life_cycle_query_dict(query=query).get(
+                    entity.name.__root__
+                )
+                if life_cycle_data:
+                    life_cycle = LifeCycle(
+                        created=AccessDetails(
+                            timestamp=convert_timestamp_to_milliseconds(
+                                life_cycle_data.created_at.timestamp()
+                            )
+                        )
+                    )
+                    yield Either(
+                        right=OMetaLifeCycleData(entity=entity, life_cycle=life_cycle)
+                    )
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name=entity.name.__root__,
+                        error=f"Unable to get the table life cycle data for table {entity.name.__root__}: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
+    def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]:
+        """
+        Get the life cycle data of the table
+        """
+        try:
+            table_fqn = fqn.build(
+                self.metadata,
+                entity_type=Table,
+                service_name=self.context.database_service,
+                database_name=self.context.database,
+                schema_name=self.context.database_schema,
+                table_name=self.context.table,
+                skip_es_search=True,
+            )
+            table = self.metadata.get_by_name(entity=Table, fqn=table_fqn)
+            if table:
+                yield from self.get_life_cycle_data(
+                    entity=table,
+                    query=self.life_cycle_query.format(
+                        database_name=table.database.name,
+                        schema_name=table.databaseSchema.name,
+                    ),
+                )
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name="lifeCycle",
+                    error=f"Error Processing life cycle data: {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )

metadata/ingestion/source/database/mongodb/connection.py

@@ -24,7 +24,6 @@ from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
 from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
-    MongoConnectionString,
     MongoDBConnection,
 )
 from metadata.ingestion.connections.builders import get_connection_url_common
@@ -36,10 +35,7 @@ def get_connection(connection: MongoDBConnection):
     """
     Create connection
     """
-    if isinstance(connection.connectionDetails, MongoConnectionString):
-        mongo_url = connection.connectionDetails.connectionURI
-    else:
-        mongo_url = get_connection_url_common(connection.connectionDetails)
+    mongo_url = get_connection_url_common(connection)
     return MongoClient(mongo_url)
 
 

metadata/ingestion/source/database/mssql/lineage.py

@@ -26,4 +26,6 @@ class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
             OR lower(t.text) LIKE '%%update%%'
             OR lower(t.text) LIKE '%%merge%%'
         )
+        AND lower(t.text) NOT LIKE '%%create%%procedure%%'
+        AND lower(t.text) NOT LIKE '%%declare%%'
     """