openmetadata-ingestion 1.8.7.0__py3-none-any.whl → 1.8.8.0__py3-none-any.whl

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-24T06:26:23+00:00
+#   timestamp: 2025-07-30T14:56:48+00:00
 
 from __future__ import annotations
 

metadata/ingestion/models/custom_pydantic.py

@@ -199,6 +199,7 @@ class _CustomSecretStr(SecretStr):
             and SecretsManagerFactory().get_secrets_manager()
         ):
             secret_id = self._secret_value.replace(SECRET, "")
+            logger.info(f"Getting secret value for {secret_id}")
             try:
                 return (
                     SecretsManagerFactory()

metadata/ingestion/ometa/ometa_api.py

@@ -59,7 +59,12 @@ from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
 from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
 from metadata.ingestion.ometa.models import EntityList
 from metadata.ingestion.ometa.routes import ROUTES
-from metadata.ingestion.ometa.utils import get_entity_type, model_str, quote
+from metadata.ingestion.ometa.utils import (
+    decode_jwt_token,
+    get_entity_type,
+    model_str,
+    quote,
+)
 from metadata.utils.logger import ometa_logger
 from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
 from metadata.utils.ssl_registry import get_verify_ssl_fn
@@ -156,6 +161,7 @@ class OpenMetadata(
         ).get_secrets_manager()
 
         self._auth_provider = OpenMetadataAuthenticationProvider.create(self.config)
+        self.log_user_name_from_jwt_token()
 
         get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
 
@@ -178,6 +184,29 @@ class OpenMetadata(
         if self.config.enableVersionValidation:
             self.validate_versions()
 
+    def log_user_name_from_jwt_token(self) -> None:
+        """
+        Log user name from JWT token.
+        """
+        # Log user name from JWT token if authProvider is openmetadata
+        if (
+            self.config.authProvider
+            and self.config.authProvider.value == "openmetadata"
+        ):
+            try:
+                # Get the JWT token from the auth provider
+                jwt_token, _ = self._auth_provider.get_access_token()
+                if jwt_token:
+                    # Decode the JWT token to extract user information
+                    payload = decode_jwt_token(jwt_token)
+                    if payload:
+                        if payload.get("sub"):
+                            logger.debug(f"Authenticated user: {payload.get('sub')}")
+                        else:
+                            logger.debug("Could not extract user name from JWT token")
+            except Exception as e:
+                logger.debug(f"Error processing JWT token: {e}")
+
     @classmethod
     def from_env(cls) -> "OpenMetadata":
         settings = OpenMetadataSettings()

metadata/ingestion/ometa/utils.py

@@ -12,9 +12,11 @@
 Helper functions to handle OpenMetadata Entities' properties
 """
 
+import base64
+import json
 import re
 import string
-from typing import Any, Type, TypeVar, Union
+from typing import Any, Dict, Optional, Type, TypeVar, Union
 
 from pydantic import BaseModel
 from requests.utils import quote as url_quote
@@ -98,3 +100,32 @@ def build_entity_reference(entity: T) -> EntityReference:
         description=entity.description,
         href=entity.href,
     )
+
+
+def decode_jwt_token(jwt_token: str) -> Optional[Dict[str, Any]]:
+    """
+    Decode JWT token to extract payload without verification.
+    JWT tokens have three parts: header.payload.signature
+    We only decode the payload part to get user information.
+    """
+    from metadata.utils.logger import ometa_logger
+
+    logger = ometa_logger()
+    try:
+        # Split the JWT token into its three parts
+        parts = jwt_token.split(".")
+        if len(parts) != 3:
+            logger.debug("Invalid JWT token format: expected 3 parts")
+            return None
+
+        # Decode the payload (second part)
+        payload = parts[1]
+        # Add padding if needed for base64 decoding
+        payload += "=" * (4 - len(payload) % 4) if len(payload) % 4 else ""
+
+        # Decode base64 and parse JSON
+        decoded_payload = base64.urlsafe_b64decode(payload)
+        return json.loads(decoded_payload.decode("utf-8"))
+    except Exception as e:
+        logger.debug(f"Failed to decode JWT token: {e}")
+        return None

metadata/ingestion/source/dashboard/tableau/client.py

@@ -41,6 +41,7 @@ from metadata.ingestion.source.dashboard.tableau.queries import (
     TALEAU_GET_CUSTOM_SQL_QUERY,
 )
 from metadata.utils.logger import ometa_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ometa_logger()
 
@@ -80,6 +81,7 @@ class TableauClient:
         config,
         verify_ssl: Union[bool, str],
         pagination_limit: int,
+        ssl_manager: Optional[SSLManager] = None,
     ):
         self.tableau_server = Server(str(config.hostPort), use_server_version=True)
         if config.apiVersion:
@@ -91,6 +93,7 @@ class TableauClient:
         self.custom_sql_table_queries: Dict[str, List[str]] = {}
         self.owner_cache: Dict[str, TableauOwner] = {}
         self.all_projects: List[ProjectItem] = []
+        self.ssl_manager = ssl_manager
 
     @cached_property
     def server_info(self) -> Callable:
@@ -414,3 +417,11 @@ class TableauClient:
 
     def sign_out(self) -> None:
         self.tableau_server.auth.sign_out()
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        """
+        Clean up temporary SSL files if they exist
+        """
+        if self.ssl_manager:
+            self.ssl_manager.cleanup_temp_files()

metadata/ingestion/source/dashboard/tableau/connection.py

@@ -38,6 +38,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
 from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
+from metadata.utils.ssl_manager import SSLManager
 
 logger = ingestion_logger()
 
@@ -47,13 +48,14 @@ def get_connection(connection: TableauConnection) -> TableauClient:
     Create connection
     """
     tableau_server_auth = build_server_config(connection)
-    verify_ssl = set_verify_ssl(connection)
+    verify_ssl, ssl_manager = set_verify_ssl(connection)
     try:
         return TableauClient(
             tableau_server_auth=tableau_server_auth,
             config=connection,
             verify_ssl=verify_ssl,
             pagination_limit=connection.paginationLimit,
+            ssl_manager=ssl_manager,
         )
     except Exception as exc:
         logger.debug(traceback.format_exc())
@@ -62,19 +64,46 @@ def get_connection(connection: TableauConnection) -> TableauClient:
         )
 
 
-def set_verify_ssl(connection: TableauConnection) -> Union[bool, str]:
+def set_verify_ssl(
+    connection: TableauConnection,
+) -> tuple[Union[bool, str], Optional[SSLManager]]:
     """
     Set verify ssl based on connection configuration
     ref: https://tableau.github.io/server-client-python/docs/sign-in-out#handling-ssl-certificates-for-tableau-server
     """
-    if connection.verifySSL.value == "validate":
-        if connection.sslConfig.root.caCertificate:
-            return connection.sslConfig.root.caCertificate.get_secret_value()
-        if connection.sslConfig.root.sslCertificate:
-            return connection.sslConfig.root.sslCertificate.get_secret_value()
+    if connection.verifySSL.value == "no-ssl":
+        return None, None
+
     if connection.verifySSL.value == "ignore":
-        return False
-    return None
+        return False, None
+
+    if connection.verifySSL.value == "validate":
+        # Use SSLManager to create temporary certificate files
+        if not connection.sslConfig:
+            raise ValueError(
+                "SSL Config is required when verifySSL is set to 'validate'. "
+                "Please provide CA certificate, SSL certificate, or SSL key."
+            )
+
+        # Create SSLManager to handle certificate files
+        ssl_manager = SSLManager(
+            ca=connection.sslConfig.root.caCertificate,
+            cert=connection.sslConfig.root.sslCertificate,
+            key=connection.sslConfig.root.sslKey,
+        )
+
+        # Return the CA certificate file path for verification
+        # If no CA certificate is provided, use default verification
+        if ssl_manager.ca_file_path:
+            return ssl_manager.ca_file_path, ssl_manager
+        else:
+            # If no CA certificate is provided but SSL is enabled, use default verification
+            return True, ssl_manager
+
+    raise ValueError(
+        f"Unsupported verifySSL value: {connection.verifySSL.value}. "
+        "Expected one of ['no-ssl', 'ignore', 'validate']."
+    )
 
 
 def test_connection(

metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -696,7 +696,7 @@ class TableauSource(DashboardServiceSource):
                                         "No table entities found for custom SQL lineage."
                                         f"fqn_search_string={fqn_search_string}, table_name={table_name}, query={query}"
                                     )
-                                for table_entity in from_entities:
+                                for table_entity in from_entities or []:
                                     yield self._get_add_lineage_request(
                                         to_entity=upstream_data_model_entity,
                                         from_entity=table_entity,

metadata/ingestion/source/database/athena/metadata.py

@@ -115,6 +115,7 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         )
         self.external_location_map = {}
         self.schema_description_map = {}
+        self.glue_client = None
 
     def prepare(self):
         """
@@ -122,8 +123,10 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         """
         try:
             super().prepare()
-            glue_client = AWSClient(self.service_connection.awsConfig).get_glue_client()
-            paginator = glue_client.get_paginator("get_databases")
+            self.glue_client = AWSClient(
+                self.service_connection.awsConfig
+            ).get_glue_client()
+            paginator = self.glue_client.get_paginator("get_databases")
             for page in paginator.paginate():
                 database_page = DatabasePage(**page)
                 for database in database_page.DatabaseList or []:
@@ -304,3 +307,23 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
         else:
             description = table_info.get("text")
         return description
+
+    def _get_columns_internal(
+        self,
+        schema_name: str,
+        table_name: str,
+        db_name: str,
+        inspector: Inspector,
+        table_type: TableType = None,
+    ):
+        """
+        Override to pass Glue client to get_columns for Iceberg table filtering
+        """
+        # Pass the Glue client as a keyword argument to get_columns
+        return inspector.get_columns(
+            table_name,
+            schema_name,
+            table_type=table_type,
+            db_name=db_name,
+            glue_client=self.glue_client,
+        )

metadata/ingestion/source/database/athena/utils.py

@@ -20,9 +20,12 @@ from sqlalchemy.engine import reflection
 
 from metadata.ingestion.source import sqa_types
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
+from metadata.utils.logger import utils_logger
 from metadata.utils.sqlalchemy_utils import is_complex_type
 
+logger = utils_logger()
 
+# pylint: disable=protected-access
 @reflection.cache
 def _get_column_type(self, type_):
     """
@@ -30,7 +33,7 @@ def _get_column_type(self, type_):
     to add custom SQA typing.
     """
     type_ = type_.replace(" ", "").lower()
-    match = self._pattern_column_type.match(type_)  # pylint: disable=protected-access
+    match = self._pattern_column_type.match(type_)
     if match:
         name = match.group(1).lower()
         length = match.group(2)
@@ -113,18 +116,17 @@ def _get_projection_details(
     return columns
 
 
+# pylint: disable=too-many-locals
 @reflection.cache
 def get_columns(self, connection, table_name, schema=None, **kw):
     """
     Method to handle table columns
     """
-    metadata = self._get_table(  # pylint: disable=protected-access
-        connection, table_name, schema=schema, **kw
-    )
+    metadata = self._get_table(connection, table_name, schema=schema, **kw)
     columns = [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,
@@ -147,10 +149,65 @@ def get_columns(self, connection, table_name, schema=None, **kw):
         columns = _get_projection_details(columns, projection_parameters)
         return columns
 
+    # Check if this is an Iceberg table
+    if metadata.parameters.get("table_type") == "ICEBERG":
+        # For Iceberg tables, get the full table metadata from Glue to access column parameters
+        try:
+            # Get the raw connection to access schema information
+            raw_connection = self._raw_connection(connection)
+            schema = schema if schema else raw_connection.schema_name
+
+            # Use the provided Glue client or create one with default credentials
+            glue_client = kw.get("glue_client")
+
+            # Get full table metadata from Glue
+            response = glue_client.get_table(DatabaseName=schema, Name=table_name)
+
+            table_info = response["Table"]
+
+            # Filter out non-current Iceberg columns
+            current_columns = []
+            storage_descriptor = table_info.get("StorageDescriptor", {})
+            glue_columns = storage_descriptor.get("Columns", [])
+
+            for glue_col in glue_columns:
+                col_name = glue_col["Name"]
+                col_type = glue_col["Type"]
+                col_comment = glue_col.get("Comment", "")
+                col_parameters = glue_col.get("Parameters", {})
+
+                # Check if this is a non-current Iceberg column
+                iceberg_current = col_parameters.get("iceberg.field.current", "true")
+                is_current = iceberg_current != "false"
+
+                if is_current:
+                    current_columns.append(
+                        {
+                            "name": col_name,
+                            "type": self._get_column_type(col_type),
+                            "nullable": True,
+                            "default": None,
+                            "autoincrement": False,
+                            "comment": col_comment,
+                            "system_data_type": col_type,
+                            "is_complex": is_complex_type(col_type),
+                            "dialect_options": {"awsathena_partition": None},
+                        }
+                    )
+
+            columns += current_columns
+            return columns
+
+        except Exception as e:
+            # If we can't get Glue metadata, fall back to the original method
+            # This ensures backward compatibility
+            logger.warning(f"Error getting Glue metadata for table {table_name}: {e}")
+
+    # For non-Iceberg tables or if Glue access fails, use the original method
     columns += [
         {
             "name": c.name,
-            "type": self._get_column_type(c.type),  # pylint: disable=protected-access
+            "type": self._get_column_type(c.type),
             "nullable": True,
             "default": None,
             "autoincrement": False,

metadata/ingestion/source/database/column_type_parser.py

@@ -444,8 +444,13 @@ class ColumnTypeParser:
         if not data_type:
             return {"dataType": "NULL", "dataTypeDisplay": dtype}
         data_length = 1
-        if re.match(r".*(\([\w]*\))", dtype):
-            data_length = re.match(r".*\(([\w]*)\)", dtype).groups()[0]
+        match = re.match(r".*\(([\w]*)\)", dtype)
+        if match:
+            try:
+                data_length = int(match.groups()[0])
+            except ValueError:
+                # If conversion fails, keep it as 1 (default)
+                data_length = 1
         return {
             "dataType": data_type,
             "dataTypeDisplay": data_type,