openmetadata-ingestion 1.8.7.1__py3-none-any.whl → 1.8.8.0__py3-none-any.whl

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-28T11:29:07+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/dashboard/tableau/client.py

@@ -41,6 +41,7 @@ from metadata.ingestion.source.dashboard.tableau.queries import (
     TALEAU_GET_CUSTOM_SQL_QUERY,
 )
 from metadata.utils.logger import ometa_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ometa_logger()
 
@@ -80,6 +81,7 @@ class TableauClient:
         config,
         verify_ssl: Union[bool, str],
         pagination_limit: int,
+        ssl_manager: Optional[SSLManager] = None,
     ):
         self.tableau_server = Server(str(config.hostPort), use_server_version=True)
         if config.apiVersion:
@@ -91,6 +93,7 @@ class TableauClient:
         self.custom_sql_table_queries: Dict[str, List[str]] = {}
         self.owner_cache: Dict[str, TableauOwner] = {}
         self.all_projects: List[ProjectItem] = []
+        self.ssl_manager = ssl_manager
 
     @cached_property
     def server_info(self) -> Callable:
@@ -414,3 +417,11 @@ class TableauClient:
 
     def sign_out(self) -> None:
         self.tableau_server.auth.sign_out()
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        """
+        Clean up temporary SSL files if they exist
+        """
+        if self.ssl_manager:
+            self.ssl_manager.cleanup_temp_files()

metadata/ingestion/source/dashboard/tableau/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ingestion_logger()
 
@@ -47,13 +48,14 @@ def get_connection(connection: TableauConnection) -> TableauClient:
     Create connection
     """
     tableau_server_auth = build_server_config(connection)
-    verify_ssl = set_verify_ssl(connection)
+    verify_ssl, ssl_manager = set_verify_ssl(connection)
     try:
         return TableauClient(
             tableau_server_auth=tableau_server_auth,
             config=connection,
             verify_ssl=verify_ssl,
             pagination_limit=connection.paginationLimit,
+            ssl_manager=ssl_manager,
         )
     except Exception as exc:
         logger.debug(traceback.format_exc())
@@ -62,19 +64,46 @@ def get_connection(connection: TableauConnection) -> TableauClient:
         )
 
 
-def set_verify_ssl(connection: TableauConnection) -> Union[bool, str]:
+def set_verify_ssl(
+    connection: TableauConnection,
+) -> tuple[Union[bool, str], Optional[SSLManager]]:
     """
     Set verify ssl based on connection configuration
     ref: https://tableau.github.io/server-client-python/docs/sign-in-out#handling-ssl-certificates-for-tableau-server
     """
-    if connection.verifySSL.value == "validate":
-        if connection.sslConfig.root.caCertificate:
-            return connection.sslConfig.root.caCertificate.get_secret_value()
-        if connection.sslConfig.root.sslCertificate:
-            return connection.sslConfig.root.sslCertificate.get_secret_value()
+    if connection.verifySSL.value == "no-ssl":
+        return None, None
+
     if connection.verifySSL.value == "ignore":
-        return False
-    return None
+        return False, None
+
+    if connection.verifySSL.value == "validate":
+        # Use SSLManager to create temporary certificate files
+        if not connection.sslConfig:
+            raise ValueError(
+                "SSL Config is required when verifySSL is set to 'validate'. "
+                "Please provide CA certificate, SSL certificate, or SSL key."
+            )
+
+        # Create SSLManager to handle certificate files
+        ssl_manager = SSLManager(
+            ca=connection.sslConfig.root.caCertificate,
+            cert=connection.sslConfig.root.sslCertificate,
+            key=connection.sslConfig.root.sslKey,
+        )
+
+        # Return the CA certificate file path for verification
+        # If no CA certificate is provided, use default verification
+        if ssl_manager.ca_file_path:
+            return ssl_manager.ca_file_path, ssl_manager
+        else:
+            # If no CA certificate is provided but SSL is enabled, use default verification
+            return True, ssl_manager
+
+    raise ValueError(
+        f"Unsupported verifySSL value: {connection.verifySSL.value}. "
+        "Expected one of ['no-ssl', 'ignore', 'validate']."
+    )
 
 
 def test_connection(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -696,7 +696,7 @@ class TableauSource(DashboardServiceSource):
                                         "No table entities found for custom SQL lineage."
                                         f"fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
                                     )
-                                for table_entity in from_entities:
+                                for table_entity in from_entities or []:
                                     yield self._get_add_lineage_request(
                                         to_entity=upstream_data_model_entity,
                                         from_entity=table_entity,

metadata/ingestion/source/database/athena/metadata.py

@@ -115,6 +115,7 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         )
         self.external_location_map = {}
         self.schema_description_map = {}
+        self.glue_client = None
 
     def prepare(self):
         """
@@ -122,8 +123,10 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         """
         try:
             super().prepare()
-            glue_client = AWSClient(self.service_connection.awsConfig).get_glue_client()
-            paginator = glue_client.get_paginator("get_databases")
+            self.glue_client = AWSClient(
+                self.service_connection.awsConfig
+            ).get_glue_client()
+            paginator = self.glue_client.get_paginator("get_databases")
             for page in paginator.paginate():
                 database_page = DatabasePage(**page)
                 for database in database_page.DatabaseList or []:
@@ -304,3 +307,23 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         else:
             description = table_info.get("text")
         return description
+
+    def _get_columns_internal(
+        self,
+        schema_name: str,
+        table_name: str,
+        db_name: str,
+        inspector: Inspector,
+        table_type: TableType = None,
+    ):
+        """
+        Override to pass Glue client to get_columns for Iceberg table filtering
+        """
+        # Pass the Glue client as a keyword argument to get_columns
+        return inspector.get_columns(
+            table_name,
+            schema_name,
+            table_type=table_type,
+            db_name=db_name,
+            glue_client=self.glue_client,
+        )

metadata/ingestion/source/database/athena/utils.py

@@ -20,9 +20,12 @@ from sqlalchemy.engine import reflection
 
 from metadata.ingestion.source import sqa_types
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
+from metadata.utils.logger import utils_logger
 from metadata.utils.sqlalchemy_utils import is_complex_type
 
+logger = utils_logger()
 
+# pylint: disable=protected-access
 @reflection.cache
 def _get_column_type(self, type_):
     """
@@ -30,7 +33,7 @@ def _get_column_type(self, type_):
     to add custom SQA typing.
     """
     type_ = type_.replace(" ", "").lower()
-    match = self._pattern_column_type.match(type_)  # pylint: disable=protected-access
+    match = self._pattern_column_type.match(type_)
     if match:
         name = match.group(1).lower()
         length = match.group(2)
@@ -113,18 +116,17 @@ def _get_projection_details(
     return columns
 
 
+# pylint: disable=too-many-locals
 @reflection.cache
 def get_columns(self, connection, table_name, schema=None, **kw):
     """
     Method to handle table columns
     """
-    metadata = self._get_table(  # pylint: disable=protected-access
-        connection, table_name, schema=schema, **kw
-    )
+    metadata = self._get_table(connection, table_name, schema=schema, **kw)
     columns = [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,
@@ -147,10 +149,65 @@ def get_columns(self, connection, table_name, schema=None, **kw):
         columns = _get_projection_details(columns, projection_parameters)
         return columns
 
+    # Check if this is an Iceberg table
+    if metadata.parameters.get("table_type") == "ICEBERG":
+        # For Iceberg tables, get the full table metadata from Glue to access column parameters
+        try:
+            # Get the raw connection to access schema information
+            raw_connection = self._raw_connection(connection)
+            schema = schema if schema else raw_connection.schema_name
+
+            # Use the provided Glue client or create one with default credentials
+            glue_client = kw.get("glue_client")
+
+            # Get full table metadata from Glue
+            response = glue_client.get_table(DatabaseName=schema, Name=table_name)
+
+            table_info = response["Table"]
+
+            # Filter out non-current Iceberg columns
+            current_columns = []
+            storage_descriptor = table_info.get("StorageDescriptor", {})
+            glue_columns = storage_descriptor.get("Columns", [])
+
+            for glue_col in glue_columns:
+                col_name = glue_col["Name"]
+                col_type = glue_col["Type"]
+                col_comment = glue_col.get("Comment", "")
+                col_parameters = glue_col.get("Parameters", {})
+
+                # Check if this is a non-current Iceberg column
+                iceberg_current = col_parameters.get("iceberg.field.current", "true")
+                is_current = iceberg_current != "false"
+
+                if is_current:
+                    current_columns.append(
+                        {
+                            "name": col_name,
+                            "type": self._get_column_type(col_type),
+                            "nullable": True,
+                            "default": None,
+                            "autoincrement": False,
+                            "comment": col_comment,
+                            "system_data_type": col_type,
+                            "is_complex": is_complex_type(col_type),
+                            "dialect_options": {"awsathena_partition": None},
+                        }
+                    )
+
+            columns += current_columns
+            return columns
+
+        except Exception as e:
+            # If we can't get Glue metadata, fall back to the original method
+            # This ensures backward compatibility
+            logger.warning(f"Error getting Glue metadata for table {table_name}: {e}")
+
+    # For non-Iceberg tables or if Glue access fails, use the original method
     columns += [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,

metadata/ingestion/source/database/column_type_parser.py

@@ -444,8 +444,13 @@ class ColumnTypeParser:
         if not data_type:
             return {"dataType": "NULL", "dataTypeDisplay": dtype}
         data_length = 1
-        if re.match(r".*(\([\w]*\))", dtype):
-            data_length = re.match(r".*\(([\w]*)\)", dtype).groups()[0]
+        match = re.match(r".*\(([\w]*)\)", dtype)
+        if match:
+            try:
+                data_length = int(match.groups()[0])
+            except ValueError:
+                # If conversion fails, keep it as 1 (default)
+                data_length = 1
         return {
             "dataType": data_type,
             "dataTypeDisplay": data_type,

metadata/ingestion/source/database/glue/metadata.py

@@ -377,12 +377,87 @@ class GlueSource(ExternalTableLineageMixin, DatabaseServiceSource):
         parsed_string["description"] = column.Comment
         return Column(**parsed_string)
 
+    # pylint: disable=too-many-locals
     def get_columns(self, column_data: StorageDetails) -> Optional[Iterable[Column]]:
+        """
+        Get columns from Glue.
+        """
+        # Check if this is an Iceberg table
+        table = self.context.get().table_data
+        is_iceberg = table.Parameters and table.Parameters.table_type == "ICEBERG"
+
+        if is_iceberg:
+            # For Iceberg tables, get the full table metadata from Glue to access column parameters
+            try:
+                schema_name = self.context.get().database_schema
+                table_name = table.Name
+
+                # Get full table metadata from Glue API
+                response = self.glue.get_table(
+                    DatabaseName=schema_name, Name=table_name
+                )
+
+                table_info = response["Table"]
+
+                # Filter out non-current Iceberg columns
+                storage_descriptor = table_info.get("StorageDescriptor", {})
+                glue_columns = storage_descriptor.get("Columns", [])
+
+                for glue_col in glue_columns:
+                    col_name = glue_col["Name"]
+                    col_type = glue_col["Type"]
+                    col_comment = glue_col.get("Comment", "")
+                    col_parameters = glue_col.get("Parameters", {})
+
+                    # Check if this is a non-current Iceberg column
+                    iceberg_current = col_parameters.get(
+                        "iceberg.field.current", "true"
+                    )
+                    is_current = iceberg_current != "false"
+
+                    if is_current:
+                        # Create a GlueColumn object for processing
+                        column_obj = GlueColumn(
+                            Name=col_name, Type=col_type, Comment=col_comment
+                        )
+                        yield self._get_column_object(column_obj)
+
+                # Process partition columns
+                partition_keys = table_info.get("PartitionKeys", [])
+                for glue_col in partition_keys:
+                    col_name = glue_col["Name"]
+                    col_type = glue_col["Type"]
+                    col_comment = glue_col.get("Comment", "")
+                    col_parameters = glue_col.get("Parameters", {})
+
+                    # Check if this is a non-current Iceberg column
+                    iceberg_current = col_parameters.get(
+                        "iceberg.field.current", "true"
+                    )
+                    is_current = iceberg_current != "false"
+
+                    if is_current:
+                        # Create a GlueColumn object for processing
+                        column_obj = GlueColumn(
+                            Name=col_name, Type=col_type, Comment=col_comment
+                        )
+                        yield self._get_column_object(column_obj)
+
+                return
+
+            except Exception as e:
+                # If we can't get Glue metadata, fall back to the original method
+                # This ensures backward compatibility
+                logger.warning(
+                    f"Failed to get Glue metadata for Iceberg table {table.Name}: {e}"
+                )
+
+        # For non-Iceberg tables or if Glue access fails, use the original method
         # process table regular columns info
         for column in column_data.Columns:
             yield self._get_column_object(column)
 
-        # process table regular columns info
+        # process table partition columns info
         for column in self.context.get().table_data.PartitionKeys:
             yield self._get_column_object(column)
 

metadata/profiler/interface/sqlalchemy/profiler_interface.py

@@ -16,6 +16,7 @@ supporting sqlalchemy abstraction layer
 """
 
 import concurrent.futures
+import gc
 import math
 import threading
 import time
@@ -26,7 +27,7 @@ from typing import Any, Dict, List, Optional, Type, Union
 
 from sqlalchemy import Column, inspect, text
 from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
-from sqlalchemy.orm import Session, scoped_session
+from sqlalchemy.orm import scoped_session
 
 from metadata.generated.schema.entity.data.table import (
     CustomMetricProfile,
@@ -461,6 +462,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
                     logger.error(error)
                     self.status.failed_profiler(error, traceback.format_exc())
                     break
+                finally:
+                    # Force garbage collection to help with memory management
+                    gc.collect()
 
         # If we've exhausted all retries without success, return a tuple of None values
         return None, None, None

metadata/readers/dataframe/base.py

@@ -22,6 +22,8 @@ from metadata.readers.file.config_source_factory import get_reader
 from metadata.readers.models import ConfigSource
 from metadata.utils.logger import ingestion_logger
 
+MAX_FILE_SIZE_FOR_PREVIEW = 50 * 1024 * 1024  # 50MB
+
 logger = ingestion_logger()