openmetadata-ingestion 1.8.7.1__py3-none-any.whl → 1.8.9.0__py3-none-any.whl

metadata/ingestion/api/parser.py

@@ -53,6 +53,10 @@ from metadata.generated.schema.entity.services.searchService import (
     SearchConnection,
     SearchServiceType,
 )
+from metadata.generated.schema.entity.services.securityService import (
+    SecurityConnection,
+    SecurityServiceType,
+)
 from metadata.generated.schema.entity.services.storageService import (
     StorageConnection,
     StorageServiceType,
@@ -115,6 +119,10 @@ from metadata.generated.schema.metadataIngestion.searchServiceMetadataPipeline i
     SearchMetadataConfigType,
     SearchServiceMetadataPipeline,
 )
+from metadata.generated.schema.metadataIngestion.securityServiceMetadataPipeline import (
+    SecurityMetadataConfigType,
+    SecurityServiceMetadataPipeline,
+)
 from metadata.generated.schema.metadataIngestion.storageServiceMetadataPipeline import (
     StorageMetadataConfigType,
     StorageServiceMetadataPipeline,
@@ -144,6 +152,7 @@ SERVICE_TYPE_MAP = {
     **{service: MlModelConnection for service in MlModelServiceType.__members__},
     **{service: StorageConnection for service in StorageServiceType.__members__},
     **{service: SearchConnection for service in SearchServiceType.__members__},
+    **{service: SecurityConnection for service in SecurityServiceType.__members__},
 }
 
 SOURCE_CONFIG_CLASS_MAP = {
@@ -157,6 +166,7 @@ SOURCE_CONFIG_CLASS_MAP = {
     DatabaseMetadataConfigType.DatabaseMetadata.value: DatabaseServiceMetadataPipeline,
     StorageMetadataConfigType.StorageMetadata.value: StorageServiceMetadataPipeline,
     SearchMetadataConfigType.SearchMetadata.value: SearchServiceMetadataPipeline,
+    SecurityMetadataConfigType.SecurityMetadata.value: SecurityServiceMetadataPipeline,
     DbtConfigType.DBT.value: DbtPipeline,
 }
 
@@ -270,25 +280,31 @@ def _parse_validation_err(validation_error: ValidationError) -> str:
     Convert the validation error into a message to log
     """
     missing_fields = [
-        f"Extra parameter '{err.get('loc')[0]}'"
-        if len(err.get("loc")) == 1
-        else f"Extra parameter in {err.get('loc')}"
+        (
+            f"Extra parameter '{err.get('loc')[0]}'"
+            if len(err.get("loc")) == 1
+            else f"Extra parameter in {err.get('loc')}"
+        )
         for err in validation_error.errors()
         if err.get("type") == "extra_forbidden"
     ]
 
     extra_fields = [
-        f"Missing parameter '{err.get('loc')[0]}'"
-        if len(err.get("loc")) == 1
-        else f"Missing parameter in {err.get('loc')}"
+        (
+            f"Missing parameter '{err.get('loc')[0]}'"
+            if len(err.get("loc")) == 1
+            else f"Missing parameter in {err.get('loc')}"
+        )
         for err in validation_error.errors()
         if err.get("type") == "missing"
     ]
 
     invalid_fields = [
-        f"Invalid parameter value for '{err.get('loc')[0]}'"
-        if len(err.get("loc")) == 1
-        else f"Invalid parameter value for {err.get('loc')}"
+        (
+            f"Invalid parameter value for '{err.get('loc')[0]}'"
+            if len(err.get("loc")) == 1
+            else f"Invalid parameter value for {err.get('loc')}"
+        )
         for err in validation_error.errors()
         if err.get("type") not in ("missing", "extra")
     ]

metadata/ingestion/ometa/routes.py

@@ -89,6 +89,9 @@ from metadata.generated.schema.api.services.createPipelineService import (
 from metadata.generated.schema.api.services.createSearchService import (
     CreateSearchServiceRequest,
 )
+from metadata.generated.schema.api.services.createSecurityService import (
+    CreateSecurityServiceRequest,
+)
 from metadata.generated.schema.api.services.createStorageService import (
     CreateStorageServiceRequest,
 )
@@ -161,6 +164,7 @@ from metadata.generated.schema.entity.services.metadataService import MetadataSe
 from metadata.generated.schema.entity.services.mlmodelService import MlModelService
 from metadata.generated.schema.entity.services.pipelineService import PipelineService
 from metadata.generated.schema.entity.services.searchService import SearchService
+from metadata.generated.schema.entity.services.securityService import SecurityService
 from metadata.generated.schema.entity.services.storageService import StorageService
 from metadata.generated.schema.entity.teams.persona import Persona
 from metadata.generated.schema.entity.teams.role import Role
@@ -256,6 +260,8 @@ ROUTES = {
     CreateMetadataServiceRequest.__name__: "/services/metadataServices",
     SearchService.__name__: "/services/searchServices",
     CreateSearchServiceRequest.__name__: "/services/searchServices",
+    SecurityService.__name__: "/services/securityServices",
+    CreateSecurityServiceRequest.__name__: "/services/securityServices",
     IngestionPipeline.__name__: "/services/ingestionPipelines",
     CreateIngestionPipelineRequest.__name__: "/services/ingestionPipelines",
     TestConnectionDefinition.__name__: "/services/testConnectionDefinitions",

metadata/ingestion/source/dashboard/tableau/client.py

@@ -41,6 +41,7 @@ from metadata.ingestion.source.dashboard.tableau.queries import (
     TALEAU_GET_CUSTOM_SQL_QUERY,
 )
 from metadata.utils.logger import ometa_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ometa_logger()
 
@@ -80,6 +81,7 @@ class TableauClient:
         config,
         verify_ssl: Union[bool, str],
         pagination_limit: int,
+        ssl_manager: Optional[SSLManager] = None,
     ):
         self.tableau_server = Server(str(config.hostPort), use_server_version=True)
         if config.apiVersion:
@@ -91,6 +93,7 @@ class TableauClient:
         self.custom_sql_table_queries: Dict[str, List[str]] = {}
         self.owner_cache: Dict[str, TableauOwner] = {}
         self.all_projects: List[ProjectItem] = []
+        self.ssl_manager = ssl_manager
 
     @cached_property
     def server_info(self) -> Callable:
@@ -414,3 +417,11 @@ class TableauClient:
 
     def sign_out(self) -> None:
         self.tableau_server.auth.sign_out()
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        """
+        Clean up temporary SSL files if they exist
+        """
+        if self.ssl_manager:
+            self.ssl_manager.cleanup_temp_files()

metadata/ingestion/source/dashboard/tableau/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ingestion_logger()
 
@@ -47,13 +48,14 @@ def get_connection(connection: TableauConnection) -> TableauClient:
     Create connection
     """
     tableau_server_auth = build_server_config(connection)
-    verify_ssl = set_verify_ssl(connection)
+    verify_ssl, ssl_manager = set_verify_ssl(connection)
     try:
         return TableauClient(
             tableau_server_auth=tableau_server_auth,
             config=connection,
             verify_ssl=verify_ssl,
             pagination_limit=connection.paginationLimit,
+            ssl_manager=ssl_manager,
         )
     except Exception as exc:
         logger.debug(traceback.format_exc())
@@ -62,19 +64,46 @@ def get_connection(connection: TableauConnection) -> TableauClient:
         )
 
 
-def set_verify_ssl(connection: TableauConnection) -> Union[bool, str]:
+def set_verify_ssl(
+    connection: TableauConnection,
+) -> tuple[Union[bool, str], Optional[SSLManager]]:
     """
     Set verify ssl based on connection configuration
     ref: https://tableau.github.io/server-client-python/docs/sign-in-out#handling-ssl-certificates-for-tableau-server
     """
-    if connection.verifySSL.value == "validate":
-        if connection.sslConfig.root.caCertificate:
-            return connection.sslConfig.root.caCertificate.get_secret_value()
-        if connection.sslConfig.root.sslCertificate:
-            return connection.sslConfig.root.sslCertificate.get_secret_value()
+    if connection.verifySSL.value == "no-ssl":
+        return None, None
+
     if connection.verifySSL.value == "ignore":
-        return False
-    return None
+        return False, None
+
+    if connection.verifySSL.value == "validate":
+        # Use SSLManager to create temporary certificate files
+        if not connection.sslConfig:
+            raise ValueError(
+                "SSL Config is required when verifySSL is set to 'validate'. "
+                "Please provide CA certificate, SSL certificate, or SSL key."
+            )
+
+        # Create SSLManager to handle certificate files
+        ssl_manager = SSLManager(
+            ca=connection.sslConfig.root.caCertificate,
+            cert=connection.sslConfig.root.sslCertificate,
+            key=connection.sslConfig.root.sslKey,
+        )
+
+        # Return the CA certificate file path for verification
+        # If no CA certificate is provided, use default verification
+        if ssl_manager.ca_file_path:
+            return ssl_manager.ca_file_path, ssl_manager
+        else:
+            # If no CA certificate is provided but SSL is enabled, use default verification
+            return True, ssl_manager
+
+    raise ValueError(
+        f"Unsupported verifySSL value: {connection.verifySSL.value}. "
+        "Expected one of ['no-ssl', 'ignore', 'validate']."
+    )
 
 
 def test_connection(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -696,7 +696,7 @@ class TableauSource(DashboardServiceSource):
                                         "No table entities found for custom SQL lineage."
                                         f"fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
                                     )
-                                for table_entity in from_entities:
+                                for table_entity in from_entities or []:
                                     yield self._get_add_lineage_request(
                                         to_entity=upstream_data_model_entity,
                                         from_entity=table_entity,

metadata/ingestion/source/database/athena/metadata.py

@@ -115,6 +115,7 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         )
         self.external_location_map = {}
         self.schema_description_map = {}
+        self.glue_client = None
 
     def prepare(self):
         """
@@ -122,8 +123,10 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         """
         try:
             super().prepare()
-            glue_client = AWSClient(self.service_connection.awsConfig).get_glue_client()
-            paginator = glue_client.get_paginator("get_databases")
+            self.glue_client = AWSClient(
+                self.service_connection.awsConfig
+            ).get_glue_client()
+            paginator = self.glue_client.get_paginator("get_databases")
             for page in paginator.paginate():
                 database_page = DatabasePage(**page)
                 for database in database_page.DatabaseList or []:
@@ -304,3 +307,23 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         else:
             description = table_info.get("text")
         return description
+
+    def _get_columns_internal(
+        self,
+        schema_name: str,
+        table_name: str,
+        db_name: str,
+        inspector: Inspector,
+        table_type: TableType = None,
+    ):
+        """
+        Override to pass Glue client to get_columns for Iceberg table filtering
+        """
+        # Pass the Glue client as a keyword argument to get_columns
+        return inspector.get_columns(
+            table_name,
+            schema_name,
+            table_type=table_type,
+            db_name=db_name,
+            glue_client=self.glue_client,
+        )

metadata/ingestion/source/database/athena/utils.py

@@ -20,9 +20,12 @@ from sqlalchemy.engine import reflection
 
 from metadata.ingestion.source import sqa_types
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
+from metadata.utils.logger import utils_logger
 from metadata.utils.sqlalchemy_utils import is_complex_type
 
+logger = utils_logger()
 
+# pylint: disable=protected-access
 @reflection.cache
 def _get_column_type(self, type_):
     """
@@ -30,7 +33,7 @@ def _get_column_type(self, type_):
     to add custom SQA typing.
     """
     type_ = type_.replace(" ", "").lower()
-    match = self._pattern_column_type.match(type_)  # pylint: disable=protected-access
+    match = self._pattern_column_type.match(type_)
     if match:
         name = match.group(1).lower()
         length = match.group(2)
@@ -113,18 +116,17 @@ def _get_projection_details(
     return columns
 
 
+# pylint: disable=too-many-locals
 @reflection.cache
 def get_columns(self, connection, table_name, schema=None, **kw):
     """
     Method to handle table columns
     """
-    metadata = self._get_table(  # pylint: disable=protected-access
-        connection, table_name, schema=schema, **kw
-    )
+    metadata = self._get_table(connection, table_name, schema=schema, **kw)
     columns = [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,
@@ -147,10 +149,65 @@ def get_columns(self, connection, table_name, schema=None, **kw):
         columns = _get_projection_details(columns, projection_parameters)
         return columns
 
+    # Check if this is an Iceberg table
+    if metadata.parameters.get("table_type") == "ICEBERG":
+        # For Iceberg tables, get the full table metadata from Glue to access column parameters
+        try:
+            # Get the raw connection to access schema information
+            raw_connection = self._raw_connection(connection)
+            schema = schema if schema else raw_connection.schema_name
+
+            # Use the provided Glue client or create one with default credentials
+            glue_client = kw.get("glue_client")
+
+            # Get full table metadata from Glue
+            response = glue_client.get_table(DatabaseName=schema, Name=table_name)
+
+            table_info = response["Table"]
+
+            # Filter out non-current Iceberg columns
+            current_columns = []
+            storage_descriptor = table_info.get("StorageDescriptor", {})
+            glue_columns = storage_descriptor.get("Columns", [])
+
+            for glue_col in glue_columns:
+                col_name = glue_col["Name"]
+                col_type = glue_col["Type"]
+                col_comment = glue_col.get("Comment", "")
+                col_parameters = glue_col.get("Parameters", {})
+
+                # Check if this is a non-current Iceberg column
+                iceberg_current = col_parameters.get("iceberg.field.current", "true")
+                is_current = iceberg_current != "false"
+
+                if is_current:
+                    current_columns.append(
+                        {
+                            "name": col_name,
+                            "type": self._get_column_type(col_type),
+                            "nullable": True,
+                            "default": None,
+                            "autoincrement": False,
+                            "comment": col_comment,
+                            "system_data_type": col_type,
+                            "is_complex": is_complex_type(col_type),
+                            "dialect_options": {"awsathena_partition": None},
+                        }
+                    )
+
+            columns += current_columns
+            return columns
+
+        except Exception as e:
+            # If we can't get Glue metadata, fall back to the original method
+            # This ensures backward compatibility
+            logger.warning(f"Error getting Glue metadata for table {table_name}: {e}")
+
+    # For non-Iceberg tables or if Glue access fails, use the original method
     columns += [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,

metadata/ingestion/source/database/column_type_parser.py

@@ -444,8 +444,13 @@ class ColumnTypeParser:
         if not data_type:
             return {"dataType": "NULL", "dataTypeDisplay": dtype}
         data_length = 1
-        if re.match(r".*(\([\w]*\))", dtype):
-            data_length = re.match(r".*\(([\w]*)\)", dtype).groups()[0]
+        match = re.match(r".*\(([\w]*)\)", dtype)
+        if match:
+            try:
+                data_length = int(match.groups()[0])
+            except ValueError:
+                # If conversion fails, keep it as 1 (default)
+                data_length = 1
         return {
             "dataType": data_type,
             "dataTypeDisplay": data_type,

metadata/ingestion/source/database/datalake/metadata.py

@@ -250,10 +250,10 @@ class DatalakeSource(DatabaseServiceSource):
         if self.source_config.includeTables:
             for key_name in self.client.get_table_names(bucket_name, prefix):
                 table_name = self.standardize_table_name(bucket_name, key_name)
-                logger.info(f"Processing table: {table_name}")
+
                 if self.filter_dl_table(table_name):
                     continue
-
+                logger.info(f"Processing table: {table_name}")
                 file_extension = get_file_format_type(
                     key_name=key_name, metadata_entry=metadata_entry
                 )

metadata/ingestion/source/database/dbt/metadata.py

@@ -147,15 +147,41 @@ class DbtSource(DbtServiceSource):
         self, manifest_node: Any, catalog_node: Optional[Any]
     ) -> Optional[EntityReferenceList]:
         """
-        Returns dbt owner
+        Returns dbt owner with priority:
+        1. manifest_node.meta.openmetadata.owner (OpenMetadata docs format - HIGHEST PRIORITY)
+        2. manifest_node.meta.owner (old format)
+        3. catalog_node.metadata.owner (standard DBT location - LOWEST PRIORITY)
         """
         try:
-            owner_ref = None
             dbt_owner = None
-            if catalog_node:
-                dbt_owner = catalog_node.metadata.owner
-            if manifest_node:
-                dbt_owner = manifest_node.meta.get(DbtCommonEnum.OWNER.value)
+
+            # PRIORITY 1: Check manifest node meta.openmetadata.owner
+            if manifest_node and manifest_node.meta:
+                openmetadata = manifest_node.meta.get("openmetadata", {})
+                if openmetadata:
+                    openmetadata_owner = openmetadata.get("owner")
+                    if openmetadata_owner:
+                        dbt_owner = openmetadata_owner
+
+            # PRIORITY 2: Check old format meta.owner
+            if not dbt_owner:
+                if manifest_node and manifest_node.meta:
+                    old_owner = manifest_node.meta.get(DbtCommonEnum.OWNER.value)
+                    if old_owner:
+                        dbt_owner = old_owner
+
+            # PRIORITY 3: Check catalog node
+            if not dbt_owner:
+                if catalog_node:
+                    try:
+                        catalog_owner = catalog_node.metadata.owner
+                        if catalog_owner:
+                            dbt_owner = catalog_owner
+                    except Exception as catalog_exc:
+                        logger.debug(
+                            f"Error accessing catalog_node.metadata.owner: {catalog_exc}"
+                        )
+
             if dbt_owner and isinstance(dbt_owner, str):
                 owner_ref = self.metadata.get_reference_by_name(
                     name=dbt_owner, is_owner=True

metadata/ingestion/source/database/glue/metadata.py

@@ -377,12 +377,87 @@ class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
         parsed_string["description"] = column.Comment
         return Column(**parsed_string)
 
+    # pylint: disable=too-many-locals
     def get_columns(self, column_data: StorageDetails) -> Optional[Iterable[Column]]:
+        """
+        Get columns from Glue.
+        """
+        # Check if this is an Iceberg table
+        table = self.context.get().table_data
+        is_iceberg = table.Parameters and table.Parameters.table_type == "ICEBERG"
+
+        if is_iceberg:
+            # For Iceberg tables, get the full table metadata from Glue to access column parameters
+            try:
+                schema_name = self.context.get().database_schema
+                table_name = table.Name
+
+                # Get full table metadata from Glue API
+                response = self.glue.get_table(
+                    DatabaseName=schema_name, Name=table_name
+                )
+
+                table_info = response["Table"]
+
+                # Filter out non-current Iceberg columns
+                storage_descriptor = table_info.get("StorageDescriptor", {})
+                glue_columns = storage_descriptor.get("Columns", [])
+
+                for glue_col in glue_columns:
+                    col_name = glue_col["Name"]
+                    col_type = glue_col["Type"]
+                    col_comment = glue_col.get("Comment", "")
+                    col_parameters = glue_col.get("Parameters", {})
+
+                    # Check if this is a non-current Iceberg column
+                    iceberg_current = col_parameters.get(
+                        "iceberg.field.current", "true"
+                    )
+                    is_current = iceberg_current != "false"
+
+                    if is_current:
+                        # Create a GlueColumn object for processing
+                        column_obj = GlueColumn(
+                            Name=col_name, Type=col_type, Comment=col_comment
+                        )
+                        yield self._get_column_object(column_obj)
+
+                # Process partition columns
+                partition_keys = table_info.get("PartitionKeys", [])
+                for glue_col in partition_keys:
+                    col_name = glue_col["Name"]
+                    col_type = glue_col["Type"]
+                    col_comment = glue_col.get("Comment", "")
+                    col_parameters = glue_col.get("Parameters", {})
+
+                    # Check if this is a non-current Iceberg column
+                    iceberg_current = col_parameters.get(
+                        "iceberg.field.current", "true"
+                    )
+                    is_current = iceberg_current != "false"
+
+                    if is_current:
+                        # Create a GlueColumn object for processing
+                        column_obj = GlueColumn(
+                            Name=col_name, Type=col_type, Comment=col_comment
+                        )
+                        yield self._get_column_object(column_obj)
+
+                return
+
+            except Exception as e:
+                # If we can't get Glue metadata, fall back to the original method
+                # This ensures backward compatibility
+                logger.warning(
+                    f"Failed to get Glue metadata for Iceberg table {table.Name}: {e}"
+                )
+
+        # For non-Iceberg tables or if Glue access fails, use the original method
         # process table regular columns info
         for column in column_data.Columns:
             yield self._get_column_object(column)
 
-        # process table regular columns info
+        # process table partition columns info
         for column in self.context.get().table_data.PartitionKeys:
             yield self._get_column_object(column)