openmetadata-ingestion 1.9.12.0__py3-none-any.whl → 1.9.13.1__py3-none-any.whl

metadata/ingestion/source/storage/s3/metadata.py

@@ -356,6 +356,7 @@ class S3Source(StorageServiceSource):
                     if entry
                     and entry.get("Key")
                     and len(entry.get("Key").split("/")) > total_depth
+                    and "/_delta_log/" not in entry.get("Key")
                 }
                 for key in candidate_keys:
                     metadata_entry_copy = deepcopy(metadata_entry)
@@ -469,7 +470,10 @@ class S3Source(StorageServiceSource):
         candidate_keys = [
             entry["Key"]
             for entry in response
-            if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+            if entry
+            and entry.get("Key")
+            and not entry.get("Key").endswith("/")
+            and "/_delta_log/" not in entry.get("Key")
         ]
         for key in candidate_keys:
             if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
@@ -678,7 +682,10 @@ class S3Source(StorageServiceSource):
                 candidate_keys = [
                     entry["Key"]
                     for entry in response[S3_CLIENT_ROOT_RESPONSE]
-                    if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+                    if entry
+                    and entry.get("Key")
+                    and not entry.get("Key").endswith("/")
+                    and "/_delta_log/" not in entry.get("Key")
                 ]
                 # pick a random key out of the candidates if any were returned
                 if candidate_keys:

metadata/parsers/json_schema_parser.py

@@ -68,9 +68,49 @@ def parse_json_schema(
 
 def get_child_models(key, value, field_models, cls: Type[BaseModel] = FieldModel):
     """
-    Method to parse the child objects in the json schema
+    Method to parse the child objects in the json schema.
+    Handles oneOf union types (e.g., Debezium CDC nullable fields).
     """
     try:
+        # Handle oneOf union types (e.g., [{"type": "null"}, {"type": "object", "properties": {...}}])
+        # Common in Debezium CDC schemas for nullable fields like "after" and "before"
+        if "oneOf" in value and isinstance(value["oneOf"], list):
+            # Find the non-null object schema in the union
+            object_schema = None
+            for option in value["oneOf"]:
+                if (
+                    isinstance(option, dict)
+                    and option.get("type") == JsonSchemaDataTypes.RECORD.value
+                ):
+                    object_schema = option
+                    break
+
+            if object_schema:
+                # Use the object schema's properties and metadata
+                cls_obj = cls(
+                    name=key,
+                    displayName=value.get("title") or object_schema.get("title"),
+                    dataType=JsonSchemaDataTypes.RECORD.name,
+                    description=value.get("description")
+                    or object_schema.get("description"),
+                )
+                children = get_json_schema_fields(
+                    object_schema.get("properties", {}), cls=cls
+                )
+                cls_obj.children = children
+                field_models.append(cls_obj)
+                return
+            # If no object found in oneOf, treat as UNKNOWN
+            cls_obj = cls(
+                name=key,
+                displayName=value.get("title"),
+                dataType=JsonSchemaDataTypes.UNKNOWN.name,
+                description=value.get("description"),
+            )
+            field_models.append(cls_obj)
+            return
+
+        # Standard type handling
         cls_obj = cls(
             name=key,
             displayName=value.get("title"),

metadata/utils/datalake/datalake_utils.py

@@ -542,8 +542,129 @@ class JsonDataFrameColumnParser(GenericDataFrameColumnParser):
         """
         if self.raw_data:
             try:
+                # First, check if this is an Iceberg/Delta Lake metadata file
+                data = json.loads(self.raw_data)
+                if self._is_iceberg_delta_metadata(data):
+                    return self._parse_iceberg_delta_schema(data)
+                # Otherwise, try to parse as standard JSON Schema
                 return parse_json_schema(schema_text=self.raw_data, cls=Column)
             except Exception as exc:
                 logger.warning(f"Unable to parse the json schema: {exc}")
                 logger.debug(traceback.format_exc())
         return self._get_columns(self.data_frame)
+
+    def _is_iceberg_delta_metadata(self, data: dict) -> bool:
+        """
+        Check if the JSON data is an Iceberg or Delta Lake metadata file.
+        These files have a specific structure with 'schema' containing 'fields'.
+        """
+        return (
+            isinstance(data, dict)
+            and "schema" in data
+            and isinstance(data["schema"], dict)
+            and "fields" in data["schema"]
+            and isinstance(data["schema"]["fields"], list)
+        )
+
+    def _parse_iceberg_delta_schema(self, data: dict) -> List[Column]:
+        """
+        Parse Iceberg/Delta Lake metadata file schema to extract columns.
+        These files have structure: {"schema": {"fields": [{"id": ..., "name": ..., "type": ..., "required": ...}, ...]}}
+        """
+        columns = []
+        schema = data.get("schema", {})
+        fields = schema.get("fields", [])
+
+        for field in fields:
+            try:
+                column_name = field.get("name", "")
+                column_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = column_type
+                if isinstance(column_type, dict):
+                    type_str = column_type.get("type", "string")
+
+                # Use DataType enum directly - it will handle the conversion
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    # If the type is not recognized, default to STRING
+                    data_type = DataType.STRING
+
+                column = Column(
+                    name=truncate_column_name(column_name),
+                    displayName=column_name,
+                    dataType=data_type,
+                    dataTypeDisplay=column_type
+                    if isinstance(column_type, str)
+                    else str(column_type),
+                )
+
+                # Handle nested struct types
+                if (
+                    isinstance(column_type, dict)
+                    and column_type.get("type") == "struct"
+                ):
+                    column.children = self._parse_struct_fields(
+                        column_type.get("fields", [])
+                    )
+                    column.dataType = DataType.STRUCT
+
+                columns.append(column)
+            except Exception as exc:
+                logger.warning(f"Unable to parse field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return columns
+
+    def _parse_struct_fields(self, fields: list) -> List[dict]:
+        """
+        Parse nested struct fields in Iceberg/Delta Lake metadata.
+        """
+        children = []
+        for field in fields:
+            try:
+                child_name = field.get("name", "")
+                child_type = field.get("type", "string")
+
+                # Get the type string from dict if needed
+                type_str = child_type
+                if isinstance(child_type, dict):
+                    type_str = child_type.get("type", "string")
+
+                # Use DataType enum directly
+                try:
+                    data_type = (
+                        DataType(type_str.upper())
+                        if isinstance(type_str, str)
+                        else DataType.STRING
+                    )
+                except (ValueError, AttributeError):
+                    data_type = DataType.STRING
+
+                child = {
+                    "name": truncate_column_name(child_name),
+                    "displayName": child_name,
+                    "dataType": data_type.value,
+                    "dataTypeDisplay": child_type
+                    if isinstance(child_type, str)
+                    else str(child_type),
+                }
+
+                # Recursively handle nested structs
+                if isinstance(child_type, dict) and child_type.get("type") == "struct":
+                    child["children"] = self._parse_struct_fields(
+                        child_type.get("fields", [])
+                    )
+
+                children.append(child)
+            except Exception as exc:
+                logger.warning(f"Unable to parse nested field {field}: {exc}")
+                logger.debug(traceback.format_exc())
+
+        return children

metadata/workflow/base.py

@@ -54,6 +54,7 @@ from metadata.utils.execution_time_tracker import ExecutionTimeTracker
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger, set_loggers_level
 from metadata.workflow.workflow_output_handler import WorkflowOutputHandler
+from metadata.workflow.workflow_resource_metrics import WorkflowResourceMetrics
 from metadata.workflow.workflow_status_mixin import WorkflowStatusMixin
 
 logger = ingestion_logger()
@@ -338,6 +339,20 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
                     f" found {len(step.status.failures)} errors"
                 )
 
+            # Only calculate resource metrics when debug is enabled
+            # for no unnecessary computations
+            if self._is_debug_enabled():
+                metrics = WorkflowResourceMetrics()
+                logger.debug(
+                    f"Workflow Resources - "
+                    f"CPU: {metrics.cpu_usage_percent:.2f}% "
+                    f"({metrics.system_cpu_cores}c/{metrics.system_cpu_threads}t) | "
+                    f"Memory: {metrics.memory_used_mb:.2f}MB/"
+                    f"{metrics.memory_total_mb:.2f}MB "
+                    f"({metrics.memory_usage_percent:.2f}%) | "
+                    f"Processes: {metrics.active_processes}"
+                )
+
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Wild exception reporting status - {exc}")

metadata/workflow/workflow_resource_metrics.py

@@ -0,0 +1,111 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Module to capture resource metrics for workflow execution monitoring
+"""
+
+import os
+from typing import Optional
+
+import psutil
+
+
+class WorkflowResourceMetrics:
+    """Captures CPU and memory metrics for a workflow process and all its children."""
+
+    def __init__(self, pid: Optional[int] = None):
+        """Initialize metrics for the given process ID (defaults to current process)."""
+        self.pid: int = pid or os.getpid()
+        self._collect_metrics()
+
+    def _collect_metrics(self):
+        """Collect metrics for this process and all its children."""
+        try:
+            process = psutil.Process(self.pid)
+
+            # Get memory and CPU for this process and all its children
+            total_mem_rss = process.memory_info().rss
+            total_cpu = process.cpu_percent(interval=1)
+            process_count = 1
+            total_threads = process.num_threads()  # Get thread count for main process
+
+            # Is process is spawning other child processes via multiprocessing
+            # then, include all child processes. This is needed to get aggregated
+            # view of CPU and memory usage
+            try:
+                children = process.children(recursive=True)
+                process_count += len(children)
+                for child in children:
+                    try:
+                        total_mem_rss += child.memory_info().rss
+                        total_cpu += child.cpu_percent(interval=0.1)
+                        total_threads += (
+                            child.num_threads()
+                        )  # Add thread count for each child
+                    except (psutil.NoSuchProcess, psutil.AccessDenied):
+                        pass
+            except Exception:
+                pass
+
+            # Get actual system memory info
+            system_mem = psutil.virtual_memory()
+
+            # Get CPU core information
+            cpu_cores = psutil.cpu_count(logical=False)  # Physical cores
+            cpu_logical = psutil.cpu_count(
+                logical=True
+            )  # Logical cores (with hyperthreading)
+
+            # Store computed metrics
+            self.cpu_usage_percent: float = total_cpu
+            self.memory_used_rss_bytes: int = total_mem_rss
+            self.memory_used_mb: float = total_mem_rss / (1024 * 1024)
+            self.memory_total_mb: float = system_mem.total / (1024 * 1024)
+            self.memory_usage_percent: float = (total_mem_rss / system_mem.total) * 100
+            self.active_processes: int = process_count
+            self.active_threads: int = total_threads
+            self.system_cpu_cores: int = cpu_cores or 1
+            self.system_cpu_threads: int = cpu_logical or 1
+
+        except (psutil.NoSuchProcess, psutil.AccessDenied, Exception):
+            # Fallback values if process metrics cannot be collected
+            self.cpu_usage_percent = 0.0
+            self.memory_used_rss_bytes = 0
+            self.memory_used_mb = 0.0
+            self.memory_total_mb = 0.0
+            self.memory_usage_percent = 0.0
+            self.active_processes = 0
+            self.active_threads = 0
+            self.system_cpu_cores = 1
+            self.system_cpu_threads = 1
+
+    def to_dict(self):
+        """Return metrics as a dictionary."""
+        return {
+            "cpu_usage_percent": self.cpu_usage_percent,
+            "memory_used_rss_bytes": self.memory_used_rss_bytes,
+            "memory_used_mb": self.memory_used_mb,
+            "memory_total_mb": self.memory_total_mb,
+            "memory_usage_percent": self.memory_usage_percent,
+            "active_processes": self.active_processes,
+            "active_threads": self.active_threads,
+            "system_cpu_cores": self.system_cpu_cores,
+            "system_cpu_threads": self.system_cpu_threads,
+        }
+
+    def __str__(self):
+        """String representation of metrics."""
+        return (
+            f"CPU: {self.cpu_usage_percent:.2f}% ({self.system_cpu_cores}c/{self.system_cpu_threads}t) | "
+            f"Memory: {self.memory_used_mb:.2f}MB/{self.memory_total_mb:.2f}MB ({self.memory_usage_percent:.2f}%) | "
+            f"Processes: {self.active_processes} | Threads: {self.active_threads}"
+        )