openmetadata-ingestion 1.9.13.0__py3-none-any.whl → 1.10.1.0__py3-none-any.whl

metadata/ingestion/source/pipeline/kafkaconnect/metadata.py

@@ -13,7 +13,7 @@ KafkaConnect source to extract metadata from OM UI
 """
 import traceback
 from datetime import datetime
-from typing import Any, Iterable, List, Optional
+from typing import Iterable, Optional
 
 from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -42,23 +42,15 @@ from metadata.generated.schema.type.basic import (
     SourceUrl,
     Timestamp,
 )
-from metadata.generated.schema.type.entityLineage import (
-    ColumnLineage,
-    EntitiesEdge,
-    LineageDetails,
-)
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
 from metadata.generated.schema.type.entityLineage import Source as LineageSource
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
-from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata, T
-from metadata.ingestion.source.pipeline.kafkaconnect.client import parse_cdc_topic_name
 from metadata.ingestion.source.pipeline.kafkaconnect.models import (
-    ConnectorType,
     KafkaConnectPipelineDetails,
-    KafkaConnectTopics,
 )
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
 from metadata.utils import fqn
@@ -75,22 +67,6 @@ STATUS_MAP = {
     "UNASSIGNED": StatusType.Pending.value,
 }
 
-# CDC envelope field names used for Debezium detection and parsing
-CDC_ENVELOPE_FIELDS = {"after", "before", "op"}
-
-
-def get_field_name(field_name: Any) -> str:
-    """
-    Extract string name from FieldName object or string.
-
-    Args:
-        field_name: FieldName object with .root attribute, or plain string
-
-    Returns:
-        String representation of the field name
-    """
-    return field_name.root if hasattr(field_name, "root") else str(field_name)
-
 
 class KafkaconnectSource(PipelineServiceSource):
     """
@@ -129,11 +105,9 @@ class KafkaconnectSource(PipelineServiceSource):
                     for task in pipeline_details.tasks or []
                 ],
                 service=self.context.get().pipeline_service,
-                description=(
-                    Markdown(pipeline_details.description)
-                    if pipeline_details.description
-                    else None
-                ),
+                description=Markdown(pipeline_details.description)
+                if pipeline_details.description
+                else None,
             )
             yield Either(right=pipeline_request)
             self.register_record(pipeline_request=pipeline_request)
@@ -200,431 +174,6 @@ class KafkaconnectSource(PipelineServiceSource):
 
         return None
 
-    def _get_entity_column_fqn(self, entity: T, column_name: str) -> Optional[str]:
-        """
-        Get column FQN for any supported entity type.
-        Dispatch based on entity type.
-
-        Args:
-            entity: Table or Topic entity
-            column_name: Column/field name
-
-        Returns:
-            Fully qualified column name or None
-        """
-        if isinstance(entity, Topic):
-            return self._get_topic_field_fqn(entity, column_name)
-        elif isinstance(entity, Table):
-            return get_column_fqn(table_entity=entity, column=column_name)
-        else:
-            logger.warning(
-                f"Unsupported entity type for column FQN: {type(entity).__name__}"
-            )
-            return None
-
-    def _parse_cdc_schema_columns(self, schema_text: str) -> List[str]:
-        """
-        Parse Debezium CDC schema JSON to extract table column names.
-
-        Looks for columns in 'after' or 'before' fields within the schema,
-        handling nullable oneOf structures.
-
-        Args:
-            schema_text: Raw JSON schema string from topic
-
-        Returns:
-            List of column names, or empty list if parsing fails
-        """
-        try:
-            import json
-
-            schema_dict = json.loads(schema_text)
-
-            # Look for 'after' or 'before' field in the schema
-            for field_name in ["after", "before"]:
-                if field_name not in schema_dict.get("properties", {}):
-                    continue
-
-                field_def = schema_dict["properties"][field_name]
-
-                # Handle oneOf (nullable types)
-                if "oneOf" not in field_def:
-                    continue
-
-                for option in field_def["oneOf"]:
-                    if isinstance(option, dict) and option.get("type") == "object":
-                        columns = list(option.get("properties", {}).keys())
-                        logger.debug(
-                            f"Parsed {len(columns)} columns from CDC '{field_name}' field"
-                        )
-                        return columns
-
-        except Exception as exc:
-            logger.debug(f"Unable to parse CDC schema text: {exc}")
-
-        return []
-
-    def _extract_columns_from_entity(self, entity: T) -> List[str]:
-        """
-        Extract column/field names from Table or Topic entity.
-
-        For Debezium CDC topics, extracts columns from the 'after' or 'before' field
-        which contains the actual table structure, not the CDC envelope fields.
-
-        Args:
-            entity: Table or Topic entity
-
-        Returns:
-            List of column/field names
-        """
-        if isinstance(entity, Table):
-            return [col.name.root for col in entity.columns or []]
-
-        if hasattr(entity, "messageSchema") and entity.messageSchema:
-            schema_fields = entity.messageSchema.schemaFields or []
-
-            # Check if this is a Debezium CDC envelope structure
-            # Can be either flat (top-level: op, before, after) or nested (Envelope -> op, before, after)
-            field_names = {get_field_name(f.name) for f in schema_fields}
-            is_debezium_cdc = CDC_ENVELOPE_FIELDS.issubset(field_names)
-
-            # Fallback: Check schemaText for CDC structure if schemaFields doesn't indicate CDC
-            if not is_debezium_cdc and entity.messageSchema.schemaText:
-                try:
-                    import json
-
-                    schema_dict = json.loads(entity.messageSchema.schemaText)
-                    schema_props = schema_dict.get("properties", {})
-                    # Check if schemaText has CDC envelope fields
-                    is_debezium_cdc = CDC_ENVELOPE_FIELDS.issubset(
-                        set(schema_props.keys())
-                    )
-                except Exception:
-                    pass
-
-            logger.debug(
-                f"Topic {get_field_name(entity.name) if hasattr(entity, 'name') else 'unknown'}: field_names={field_names}, is_debezium_cdc={is_debezium_cdc}"
-            )
-
-            # Check for nested Debezium CDC structure (single Envelope field with CDC children)
-            if not is_debezium_cdc and len(schema_fields) == 1:
-                envelope_field = schema_fields[0]
-                if envelope_field.children:
-                    envelope_child_names = {
-                        get_field_name(c.name) for c in envelope_field.children
-                    }
-                    is_debezium_cdc = CDC_ENVELOPE_FIELDS.issubset(envelope_child_names)
-                    if is_debezium_cdc:
-                        logger.debug(
-                            f"Nested Debezium CDC envelope detected: {get_field_name(envelope_field.name)}"
-                        )
-                        schema_fields = (
-                            envelope_field.children
-                        )  # Use envelope children as schema fields
-
-            if is_debezium_cdc:
-                # For Debezium CDC, extract columns from the 'after' field (or 'before' as fallback)
-                # The 'after' field contains the complete record structure after the change
-                for field in schema_fields:
-                    field_name_str = get_field_name(field.name)
-                    # Prefer 'after' for source connectors (contains new/updated record state)
-                    if field_name_str == "after" and field.children:
-                        columns = [
-                            get_field_name(child.name) for child in field.children
-                        ]
-                        logger.debug(
-                            f"Debezium CDC: extracted {len(columns)} columns from 'after' field"
-                        )
-                        return columns
-
-                # Fallback to 'before' if 'after' has no children
-                for field in schema_fields:
-                    field_name_str = get_field_name(field.name)
-                    if field_name_str == "before" and field.children:
-                        columns = [
-                            get_field_name(child.name) for child in field.children
-                        ]
-                        logger.debug(
-                            f"Debezium CDC: extracted {len(columns)} columns from 'before' field"
-                        )
-                        return columns
-
-                # Final fallback: Parse schemaText if after/before don't have children
-                if entity.messageSchema.schemaText:
-                    columns = self._parse_cdc_schema_columns(
-                        entity.messageSchema.schemaText
-                    )
-                    if columns:
-                        logger.debug(
-                            f"Debezium CDC: extracted {len(columns)} columns from schemaText"
-                        )
-                        return columns
-
-                logger.debug(
-                    "Debezium CDC detected but unable to extract columns from after/before fields"
-                )
-                return []
-
-            # Non-CDC topic: extract all fields
-            columns = []
-            for field in schema_fields:
-                if field.children:
-                    columns.extend(
-                        [get_field_name(child.name) for child in field.children]
-                    )
-                else:
-                    columns.append(get_field_name(field.name))
-            return columns
-
-        return []
-
-    def _get_topic_field_fqn(
-        self, topic_entity: Topic, field_name: str
-    ) -> Optional[str]:
-        """
-        Get the fully qualified name for a field in a Topic's schema.
-        Handles nested structures where fields may be children of a parent RECORD.
-        For Debezium CDC topics, searches for fields inside after/before envelope children.
-        """
-        if (
-            not topic_entity.messageSchema
-            or not topic_entity.messageSchema.schemaFields
-        ):
-            logger.debug(
-                f"Topic {get_field_name(topic_entity.name)} has no message schema"
-            )
-            return None
-
-        # Search for the field in the schema (including nested fields)
-        for field in topic_entity.messageSchema.schemaFields:
-            field_name_str = get_field_name(field.name)
-
-            # Check if it's a direct field
-            if field_name_str == field_name:
-                return (
-                    field.fullyQualifiedName.root if field.fullyQualifiedName else None
-                )
-
-            # Check if it's a child field (nested - one level deep)
-            if field.children:
-                # For Debezium CDC, prioritize 'after' over 'before' when searching for grandchildren
-                children_to_search = field.children
-                after_child = None
-                before_child = None
-
-                for child in field.children:
-                    child_name = get_field_name(child.name)
-                    if child_name == "after":
-                        after_child = child
-                    elif child_name == "before":
-                        before_child = child
-                    # Check direct child match
-                    if child_name == field_name:
-                        return (
-                            child.fullyQualifiedName.root
-                            if child.fullyQualifiedName
-                            else None
-                        )
-
-                # Search grandchildren - prefer 'after' over 'before' for CDC topics
-                for cdc_child in [after_child, before_child]:
-                    if cdc_child and cdc_child.children:
-                        for grandchild in cdc_child.children:
-                            if get_field_name(grandchild.name) == field_name:
-                                return (
-                                    grandchild.fullyQualifiedName.root
-                                    if grandchild.fullyQualifiedName
-                                    else None
-                                )
-
-                # Search other grandchildren (non-CDC fields)
-                for child in field.children:
-                    if child not in [after_child, before_child] and child.children:
-                        for grandchild in child.children:
-                            if get_field_name(grandchild.name) == field_name:
-                                return (
-                                    grandchild.fullyQualifiedName.root
-                                    if grandchild.fullyQualifiedName
-                                    else None
-                                )
-
-        # For Debezium CDC topics, columns might only exist in schemaText (not as field objects)
-        # Manually construct FQN: topicFQN.Envelope.columnName
-        for field in topic_entity.messageSchema.schemaFields:
-            field_name_str = get_field_name(field.name)
-            # Check if this is a CDC envelope field
-            if "Envelope" in field_name_str and field.fullyQualifiedName:
-                # Construct FQN manually for CDC column
-                envelope_fqn = field.fullyQualifiedName.root
-                return f"{envelope_fqn}.{field_name}"
-
-        logger.debug(
-            f"Field {field_name} not found in topic {get_field_name(topic_entity.name)} schema"
-        )
-        return None
-
-    def build_column_lineage(
-        self,
-        from_entity: T,
-        to_entity: T,
-        topic_entity: Topic,
-        pipeline_details: KafkaConnectPipelineDetails,
-    ) -> Optional[List[ColumnLineage]]:
-        """
-        Build column-level lineage between source table, topic, and target table.
-        For source connectors: Table columns -> Topic schema fields
-        For sink connectors: Topic schema fields -> Table columns
-        """
-        try:
-            column_lineages = []
-
-            # Get column mappings from connector config if available
-            if pipeline_details.dataset and pipeline_details.dataset.column_mappings:
-                # Use explicit column mappings from connector config
-                for mapping in pipeline_details.dataset.column_mappings:
-                    if pipeline_details.conn_type == ConnectorType.SINK.value:
-                        from_col = get_column_fqn(
-                            table_entity=topic_entity, column=mapping.source_column
-                        )
-                        to_col = get_column_fqn(
-                            table_entity=to_entity, column=mapping.target_column
-                        )
-                    else:
-                        from_col = get_column_fqn(
-                            table_entity=from_entity, column=mapping.source_column
-                        )
-                        to_col = get_column_fqn(
-                            table_entity=topic_entity, column=mapping.target_column
-                        )
-
-                    if from_col and to_col:
-                        column_lineages.append(
-                            ColumnLineage(
-                                fromColumns=[from_col],
-                                toColumn=to_col,
-                                function=None,
-                            )
-                        )
-            else:
-                # Infer 1:1 column mappings based on matching column names
-                if pipeline_details.conn_type == ConnectorType.SINK.value:
-                    source_entity = topic_entity
-                    target_entity = to_entity
-                else:
-                    source_entity = from_entity
-                    target_entity = topic_entity
-
-                # Extract columns from both entities
-                source_columns = self._extract_columns_from_entity(source_entity)
-                target_columns = self._extract_columns_from_entity(target_entity)
-
-                logger.debug(
-                    f"Column matching for {pipeline_details.name}: "
-                    f"source={len(source_columns)} cols from {source_entity.__class__.__name__}, "
-                    f"target={len(target_columns)} cols from {target_entity.__class__.__name__}"
-                )
-                logger.debug(f"Source columns: {source_columns[:5]}")  # First 5
-                logger.debug(f"Target columns: {target_columns}")
-
-                # Create lookup dictionary for O(n) performance instead of O(n²)
-                target_cols_map = {str(col).lower(): col for col in target_columns}
-
-                # Match columns by name (case-insensitive)
-                for source_col_name in source_columns:
-                    source_key = str(source_col_name).lower()
-                    if source_key in target_cols_map:
-                        target_col_name = target_cols_map[source_key]
-                        logger.debug(
-                            f"Matched column: {source_col_name} -> {target_col_name}"
-                        )
-                        try:
-                            # Get fully qualified names for source and target columns
-                            from_col = self._get_entity_column_fqn(
-                                source_entity, source_col_name
-                            )
-                            to_col = self._get_entity_column_fqn(
-                                target_entity, target_col_name
-                            )
-
-                            logger.debug(f"FQNs: from_col={from_col}, to_col={to_col}")
-
-                            if from_col and to_col:
-                                column_lineages.append(
-                                    ColumnLineage(
-                                        fromColumns=[from_col],
-                                        toColumn=to_col,
-                                        function=None,
-                                    )
-                                )
-                                logger.debug(
-                                    f"Added column lineage: {from_col} -> {to_col}"
-                                )
-                        except (KeyError, AttributeError) as exc:
-                            logger.debug(
-                                f"Error creating column lineage for {source_col_name} -> {target_col_name}: {exc}"
-                            )
-
-            if column_lineages:
-                logger.debug(
-                    f"Created {len(column_lineages)} column lineages for {pipeline_details.name}"
-                )
-            return column_lineages if column_lineages else None
-
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Unable to build column lineage: {exc}")
-
-        return None
-
-    def _query_cdc_topics_from_messaging_service(
-        self, database_server_name: str
-    ) -> List[KafkaConnectTopics]:
-        """
-        Query topics from messaging service and filter by CDC naming pattern.
-
-        Used for CDC connectors without explicit topic lists - discovers topics
-        by matching against database.server.name prefix.
-
-        Args:
-            database_server_name: The database.server.name or topic.prefix from connector config
-
-        Returns:
-            List of matching CDC topics
-        """
-        topics_found = []
-
-        try:
-            logger.debug(
-                f"CDC connector without topics list - querying messaging service "
-                f"for pattern: {database_server_name}.*"
-            )
-
-            # List topics from the configured messaging service only
-            topics_list = self.metadata.list_entities(
-                entity=Topic,
-                fields=["name", "fullyQualifiedName", "service"],
-                params={"service": self.service_connection.messagingServiceName},
-            ).entities
-
-            # Filter topics that match the CDC naming pattern
-            for topic_entity in topics_list or []:
-                topic_name = (
-                    topic_entity.name.root
-                    if hasattr(topic_entity.name, "root")
-                    else str(topic_entity.name)
-                )
-
-                # Parse the topic to see if it's a CDC topic related to this connector
-                topic_info = parse_cdc_topic_name(topic_name, database_server_name)
-                if topic_info:
-                    topics_found.append(KafkaConnectTopics(name=topic_name))
-                    logger.debug(f"Matched CDC topic: {topic_name} -> {topic_info}")
-
-        except Exception as exc:
-            logger.debug(f"Unable to query topics from messaging service: {exc}")
-
-        return topics_found
-
     def yield_pipeline_lineage_details(
         self, pipeline_details: KafkaConnectPipelineDetails
     ) -> Iterable[Either[AddLineageRequest]]:
@@ -647,30 +196,14 @@ class KafkaconnectSource(PipelineServiceSource):
                 entity=Pipeline, fqn=pipeline_fqn
             )
 
-            dataset_entity = self.get_dataset_entity(pipeline_details=pipeline_details)
+            lineage_details = LineageDetails(
+                pipeline=EntityReference(id=pipeline_entity.id.root, type="pipeline"),
+                source=LineageSource.PipelineLineage,
+            )
 
-            # Get database.server.name or topic.prefix for CDC topic parsing
-            # These are ONLY set by Debezium CDC connectors
-            database_server_name = None
-            if pipeline_details.config:
-                database_server_name = pipeline_details.config.get(
-                    "database.server.name"
-                ) or pipeline_details.config.get("topic.prefix")
-
-            # For CDC connectors without explicit topics, query topics from messaging service
-            # and filter by CDC naming pattern
-            # Only do this for Debezium CDC connectors (identified by database.server.name or topic.prefix)
-            topics_to_process = pipeline_details.topics or []
-            if (
-                not topics_to_process
-                and database_server_name
-                and pipeline_details.conn_type == ConnectorType.SOURCE.value
-            ):
-                topics_to_process = self._query_cdc_topics_from_messaging_service(
-                    database_server_name
-                )
+            dataset_entity = self.get_dataset_entity(pipeline_details=pipeline_details)
 
-            for topic in topics_to_process:
+            for topic in pipeline_details.topics or []:
                 topic_fqn = fqn.build(
                     metadata=self.metadata,
                     entity_type=Topic,
@@ -680,76 +213,13 @@ class KafkaconnectSource(PipelineServiceSource):
 
                 topic_entity = self.metadata.get_by_name(entity=Topic, fqn=topic_fqn)
 
-                if topic_entity is None:
-                    continue
-
-                # If no dataset entity from config, try to parse table info from CDC topic name
-                current_dataset_entity = dataset_entity
-                if (
-                    current_dataset_entity is None
-                    and pipeline_details.conn_type == ConnectorType.SOURCE.value
-                ):
-                    # Parse CDC topic name to extract table information
-                    topic_info = parse_cdc_topic_name(
-                        str(topic.name), database_server_name
-                    )
-                    if topic_info.get("database") and topic_info.get("table"):
-                        logger.debug(
-                            f"Parsed CDC topic {topic.name}: database={topic_info['database']}, table={topic_info['table']}"
-                        )
-                        # Try to find the table entity
-                        for (
-                            dbservicename
-                        ) in self.source_config.lineageInformation.dbServiceNames or [
-                            "*"
-                        ]:
-                            table_fqn = fqn.build(
-                                metadata=self.metadata,
-                                entity_type=Table,
-                                table_name=topic_info["table"],
-                                database_name=None,
-                                schema_name=topic_info["database"],
-                                service_name=dbservicename,
-                            )
-                            current_dataset_entity = self.metadata.get_by_name(
-                                entity=Table, fqn=table_fqn
-                            )
-                            if current_dataset_entity:
-                                logger.debug(f"Found table entity: {table_fqn}")
-                                break
-
-                if current_dataset_entity is None:
-                    # No table entity found, skip this topic
+                if topic_entity is None or dataset_entity is None:
                     continue
 
-                if pipeline_details.conn_type == ConnectorType.SINK.value:
-                    from_entity, to_entity = topic_entity, current_dataset_entity
+                if pipeline_details.conn_type.lower() == "sink":
+                    from_entity, to_entity = topic_entity, dataset_entity
                 else:
-                    from_entity, to_entity = current_dataset_entity, topic_entity
-
-                # Build column-level lineage (best effort - don't fail entity-level lineage)
-                column_lineage = None
-                try:
-                    column_lineage = self.build_column_lineage(
-                        from_entity=from_entity,
-                        to_entity=to_entity,
-                        topic_entity=topic_entity,
-                        pipeline_details=pipeline_details,
-                    )
-                except Exception as exc:
-                    logger.warning(
-                        f"Failed to build column-level lineage for {pipeline_details.name}: {exc}. "
-                        "Entity-level lineage will still be created."
-                    )
-                    logger.debug(traceback.format_exc())
-
-                lineage_details = LineageDetails(
-                    pipeline=EntityReference(
-                        id=pipeline_entity.id.root, type="pipeline"
-                    ),
-                    source=LineageSource.PipelineLineage,
-                    columnsLineage=column_lineage,
-                )
+                    from_entity, to_entity = dataset_entity, topic_entity
 
                 yield Either(
                     right=AddLineageRequest(
@@ -821,7 +291,7 @@ class KafkaconnectSource(PipelineServiceSource):
                     pipeline_details.status, StatusType.Pending
                 ),
                 taskStatus=task_status,
-                timestamp=Timestamp(datetime_to_ts(datetime.now())),
+                timestamp=Timestamp(datetime_to_ts(datetime.now()))
                 # Kafka connect doesn't provide any details with exec time
             )
 

metadata/ingestion/source/pipeline/kafkaconnect/models.py

@@ -13,23 +13,14 @@
 KafkaConnect Source Model module
 """
 
-from enum import Enum
 from typing import List, Optional, Type, Union
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field
 
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.table import Table
 
 
-class ConnectorType(str, Enum):
-    """Kafka Connect connector types"""
-
-    SOURCE = "source"
-    SINK = "sink"
-    UNKNOWN = "UNKNOWN"
-
-
 class KafkaConnectTasks(BaseModel):
     id: int = Field(..., description="ID of the task")
     state: Optional[str] = Field(
@@ -44,20 +35,10 @@ class KafkaConnectTopics(BaseModel):
     name: str = Field(..., description="Name of the topic (e.g., random-source-avro)")
 
 
-class KafkaConnectColumnMapping(BaseModel):
-    """Model for column-level mapping between source and target"""
-
-    source_column: str = Field(..., description="Source column name")
-    target_column: str = Field(..., description="Target column/field name")
-
-
 class KafkaConnectDatasetDetails(BaseModel):
     table: Optional[str] = None
     database: Optional[str] = None
     container_name: Optional[str] = None
-    column_mappings: List[KafkaConnectColumnMapping] = Field(
-        default_factory=list, description="Column-level mappings if available"
-    )
 
     @property
     def dataset_type(self) -> Optional[Type[Union[Table, Container]]]:
@@ -82,15 +63,3 @@ class KafkaConnectPipelineDetails(BaseModel):
     description: Optional[str] = None
     dataset: Optional[KafkaConnectDatasetDetails] = None
     config: Optional[dict] = Field(default_factory=dict)
-
-    @field_validator("conn_type", mode="before")
-    @classmethod
-    def normalize_connector_type(cls, value: str) -> str:
-        """Normalize connector type to enum value"""
-        if value:
-            value_lower = value.lower()
-            if value_lower == "source":
-                return ConnectorType.SOURCE.value
-            elif value_lower == "sink":
-                return ConnectorType.SINK.value
-        return ConnectorType.UNKNOWN.value