openmetadata-ingestion 1.10.0.0__py3-none-any.whl → 1.10.2.0__py3-none-any.whl

metadata/ingestion/source/pipeline/kafkaconnect/models.py

@@ -13,14 +13,23 @@
 KafkaConnect Source Model module
 """
 
+from enum import Enum
 from typing import List, Optional, Type, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.table import Table
 
 
+class ConnectorType(str, Enum):
+    """Kafka Connect connector types"""
+
+    SOURCE = "source"
+    SINK = "sink"
+    UNKNOWN = "UNKNOWN"
+
+
 class KafkaConnectTasks(BaseModel):
     id: int = Field(..., description="ID of the task")
     state: Optional[str] = Field(
@@ -33,12 +42,25 @@ class KafkaConnectTasks(BaseModel):
 
 class KafkaConnectTopics(BaseModel):
     name: str = Field(..., description="Name of the topic (e.g., random-source-avro)")
+    fqn: Optional[str] = Field(
+        default=None, description="Fully qualified name of the topic in OpenMetadata"
+    )
+
+
+class KafkaConnectColumnMapping(BaseModel):
+    """Model for column-level mapping between source and target"""
+
+    source_column: str = Field(..., description="Source column name")
+    target_column: str = Field(..., description="Target column/field name")
 
 
 class KafkaConnectDatasetDetails(BaseModel):
     table: Optional[str] = None
     database: Optional[str] = None
     container_name: Optional[str] = None
+    column_mappings: List[KafkaConnectColumnMapping] = Field(
+        default_factory=list, description="Column-level mappings if available"
+    )
 
     @property
     def dataset_type(self) -> Optional[Type[Union[Table, Container]]]:
@@ -63,3 +85,15 @@ class KafkaConnectPipelineDetails(BaseModel):
     description: Optional[str] = None
     dataset: Optional[KafkaConnectDatasetDetails] = None
     config: Optional[dict] = Field(default_factory=dict)
+
+    @field_validator("conn_type", mode="before")
+    @classmethod
+    def normalize_connector_type(cls, value: str) -> str:
+        """Normalize connector type to enum value"""
+        if value:
+            value_lower = value.lower()
+            if value_lower == "source":
+                return ConnectorType.SOURCE.value
+            elif value_lower == "sink":
+                return ConnectorType.SINK.value
+        return ConnectorType.UNKNOWN.value

metadata/ingestion/source/storage/s3/metadata.py

@@ -356,6 +356,7 @@ class S3Source(StorageServiceSource):
                     if entry
                     and entry.get("Key")
                     and len(entry.get("Key").split("/")) > total_depth
+                    and "/_delta_log/" not in entry.get("Key")
                 }
                 for key in candidate_keys:
                     metadata_entry_copy = deepcopy(metadata_entry)
@@ -469,7 +470,10 @@ class S3Source(StorageServiceSource):
         candidate_keys = [
             entry["Key"]
             for entry in response
-            if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+            if entry
+            and entry.get("Key")
+            and not entry.get("Key").endswith("/")
+            and "/_delta_log/" not in entry.get("Key")
         ]
         for key in candidate_keys:
             if self.is_valid_unstructured_file(metadata_entry.unstructuredFormats, key):
@@ -678,7 +682,10 @@ class S3Source(StorageServiceSource):
                 candidate_keys = [
                     entry["Key"]
                     for entry in response[S3_CLIENT_ROOT_RESPONSE]
-                    if entry and entry.get("Key") and not entry.get("Key").endswith("/")
+                    if entry
+                    and entry.get("Key")
+                    and not entry.get("Key").endswith("/")
+                    and "/_delta_log/" not in entry.get("Key")
                 ]
                 # pick a random key out of the candidates if any were returned
                 if candidate_keys:

metadata/parsers/json_schema_parser.py

@@ -68,9 +68,49 @@ def parse_json_schema(
 
 def get_child_models(key, value, field_models, cls: Type[BaseModel] = FieldModel):
     """
-    Method to parse the child objects in the json schema
+    Method to parse the child objects in the json schema.
+    Handles oneOf union types (e.g., Debezium CDC nullable fields).
     """
     try:
+        # Handle oneOf union types (e.g., [{"type": "null"}, {"type": "object", "properties": {...}}])
+        # Common in Debezium CDC schemas for nullable fields like "after" and "before"
+        if "oneOf" in value and isinstance(value["oneOf"], list):
+            # Find the non-null object schema in the union
+            object_schema = None
+            for option in value["oneOf"]:
+                if (
+                    isinstance(option, dict)
+                    and option.get("type") == JsonSchemaDataTypes.RECORD.value
+                ):
+                    object_schema = option
+                    break
+
+            if object_schema:
+                # Use the object schema's properties and metadata
+                cls_obj = cls(
+                    name=key,
+                    displayName=value.get("title") or object_schema.get("title"),
+                    dataType=JsonSchemaDataTypes.RECORD.name,
+                    description=value.get("description")
+                    or object_schema.get("description"),
+                )
+                children = get_json_schema_fields(
+                    object_schema.get("properties", {}), cls=cls
+                )
+                cls_obj.children = children
+                field_models.append(cls_obj)
+                return
+            # If no object found in oneOf, treat as UNKNOWN
+            cls_obj = cls(
+                name=key,
+                displayName=value.get("title"),
+                dataType=JsonSchemaDataTypes.UNKNOWN.name,
+                description=value.get("description"),
+            )
+            field_models.append(cls_obj)
+            return
+
+        # Standard type handling
         cls_obj = cls(
             name=key,
             displayName=value.get("title"),

metadata/utils/deprecation.py

@@ -11,9 +11,11 @@
 """
 Announce method deprecation
 """
-import warnings
+import logging
 from functools import wraps
 
+from metadata.utils.logger import METADATA_LOGGER
+
 
 def deprecated(message: str, release: str):
     """Decorator factory to accept specific messages for each function"""
@@ -21,12 +23,13 @@ def deprecated(message: str, release: str):
     def _deprecated(fn):
         @wraps(fn)
         def inner(*args, **kwargs):
-            warnings.simplefilter("always", DeprecationWarning)
-            warnings.warn(
-                f"[{fn.__name__}] will be deprecated in the release [{release}]: {message}",
-                category=DeprecationWarning,
+            # Get the metadata logger
+            logger = logging.getLogger(METADATA_LOGGER)
+            # Log deprecation warning using the logging system
+            # This will respect the loggerLevel configuration
+            logger.warning(
+                f"[{fn.__name__}] will be deprecated in the release [{release}]: {message}"
             )
-            warnings.simplefilter("default", DeprecationWarning)
 
             return fn(*args, **kwargs)
 

metadata/utils/fqn.py

@@ -13,10 +13,12 @@ Handle FQN building and splitting logic.
 Filter information has been taken from the
 ES indexes definitions
 """
+from __future__ import annotations
+
 import hashlib
 import re
 import traceback
-from typing import Dict, List, Optional, Type, TypeVar, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Type, TypeVar, Union
 
 from antlr4.CommonTokenStream import CommonTokenStream
 from antlr4.error.ErrorStrategy import BailErrorStrategy
@@ -51,11 +53,13 @@ from metadata.generated.schema.entity.teams.team import Team
 from metadata.generated.schema.entity.teams.user import User
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testSuite import TestSuite
-from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.dispatch import class_register
 from metadata.utils.elasticsearch import get_entity_from_es_result
 from metadata.utils.logger import utils_logger
 
+if TYPE_CHECKING:
+    from metadata.ingestion.ometa.ometa_api import OpenMetadata
+
 logger = utils_logger()
 
 T = TypeVar("T", bound=BaseModel)
@@ -866,3 +870,56 @@ def get_query_checksum(query: str) -> str:
     The checksum is used as the query's name.
     """
     return hashlib.md5(query.encode()).hexdigest()
+
+
+# Not adding container since children can have recursive slots: service.container1.container2...
+FQN_ENTITY_SLOTS = {
+    Table.__name__: 4,
+    DatabaseSchema.__name__: 3,
+    Database.__name__: 2,
+    Dashboard.__name__: 2,
+    APICollection.__name__: 2,
+    Chart.__name__: 2,
+    MlModel.__name__: 2,
+    Topic.__name__: 2,
+    SearchIndex.__name__: 2,
+    Tag.__name__: 2,
+    DataModel.__name__: 2,
+    StoredProcedure.__name__: 4,
+    Pipeline.__name__: 2,
+}
+
+
+def prefix_entity_for_wildcard_search(entity_type: Type[T], fqn: str) -> str:
+    """
+    Given an entity type and an FQN, return the FQN prefixed with wildcards
+    to match any parent hierarchy leading to that entity.
+
+    For example, for a Topic with FQN "potato", return "*.potato" to match
+    the topic in any service. For a Table with FQN "schema.table", return
+    "*.*.schema.table" to match the table in any service and database.
+
+    Args:
+        entity_type: The entity type to match.
+        fqn: The FQN to prefix.
+
+    Returns:
+        The prefixed FQN with wildcards for missing parent levels.
+    """
+    slots = FQN_ENTITY_SLOTS.get(entity_type.__name__)
+    if not slots:
+        raise FQNBuildingException(
+            f"Entity type {entity_type.__name__} not supported for wildcard search"
+        )
+
+    parts = split(fqn)
+    if len(parts) > slots:
+        raise FQNBuildingException(
+            f"FQN {fqn} has too many parts ({len(parts)})"
+            f"for entity type {entity_type.__name__} (expected {slots} or fewer)"
+        )
+
+    # Add wildcards for missing parent levels
+    wildcards_needed = slots - len(parts)
+    prefixed_parts = ["*"] * wildcards_needed + parts
+    return _build(*prefixed_parts, quote=True)