openmetadata-ingestion 1.9.13.0__py3-none-any.whl → 1.10.1.0__py3-none-any.whl

metadata/ingestion/source/pipeline/kafkaconnect/client.py

@@ -14,7 +14,6 @@ Client to interact with Kafka Connect REST APIs
 
 import traceback
 from typing import List, Optional
-from urllib.parse import urlparse
 
 from kafka_connect import KafkaConnect
 
@@ -22,7 +21,6 @@ from metadata.generated.schema.entity.services.connections.pipeline.kafkaConnect
     KafkaConnectConnection,
 )
 from metadata.ingestion.source.pipeline.kafkaconnect.models import (
-    KafkaConnectColumnMapping,
     KafkaConnectDatasetDetails,
     KafkaConnectPipelineDetails,
     KafkaConnectTopics,
@@ -32,111 +30,16 @@ from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
 
-
-def parse_cdc_topic_name(topic_name: str, database_server_name: str = None) -> dict:
-    """
-    Parse CDC topic names to extract database and table information.
-
-    Common CDC topic naming patterns:
-    - Debezium: {server-name}.{database}.{table}
-    - Debezium V2: {topic-prefix}.{database}.{table}
-    - Examples:
-      - MysqlKafkaV2.ecommerce.orders -> database=ecommerce, table=orders
-      - PostgresKafkaCDC.public.orders -> database=public, table=orders
-      - ecommerce.customers -> database=ecommerce, table=customers (if server-name matches)
-
-    Args:
-        topic_name: The Kafka topic name
-        database_server_name: The database.server.name or topic.prefix from connector config
-
-    Returns:
-        dict with 'database' and 'table' keys, or empty dict if pattern doesn't match
-    """
-    if not topic_name:
-        return {}
-
-    # Skip internal/system topics
-    if topic_name.startswith(("_", "dbhistory.", "__")):
-        return {}
-
-    parts = topic_name.split(".")
-
-    # Pattern: {prefix}.{database}.{table} (3 parts)
-    if len(parts) == 3:
-        prefix, database, table = parts
-        # Verify prefix matches server name if provided
-        if database_server_name and prefix.lower() != database_server_name.lower():
-            # Might be schema.database.table for some connectors
-            pass
-        return {"database": database, "table": table}
-
-    # Pattern: {database}.{table} (2 parts)
-    elif len(parts) == 2:
-        database, table = parts
-        # Only accept if server name matches or not provided
-        if database_server_name and database.lower() == database_server_name.lower():
-            # This is server_name.table, so database is the server name
-            return {"database": database, "table": table}
-        # Or accept as database.table
-        return {"database": database, "table": table}
-
-    # Pattern: just {table} (1 part) - use server name as database
-    elif len(parts) == 1 and database_server_name:
-        return {"database": database_server_name, "table": topic_name}
-
-    return {}
-
-
-class ConnectorConfigKeys:
-    """Configuration keys for various Kafka Connect connectors"""
-
-    TABLE_KEYS = [
+SUPPORTED_DATASETS = {
+    "table": [
         "table",
         "collection",
         "snowflake.schema.name",
         "table.whitelist",
         "fields.whitelist",
-        "table.include.list",
-        "table.name.format",
-        "tables.include",
-        "table.exclude.list",
-        "snowflake.schema",
-        "snowflake.topic2table.map",
-        "fields.included",
-    ]
-
-    DATABASE_KEYS = [
-        "database",
-        "db.name",
-        "snowflake.database.name",
-        "database.include.list",
-        "database.hostname",
-        "connection.url",
-        "database.dbname",
-        "topic.prefix",
-        "database.server.name",  # Debezium V1
-        "databases.include",
-        "database.names",
-        "snowflake.database",
-        "connection.host",
-        "database.exclude.list",
-    ]
-
-    CONTAINER_KEYS = [
-        "s3.bucket.name",
-        "s3.bucket",
-        "gcs.bucket.name",
-        "azure.container.name",
-        "topics.dir",  # Directory path within storage container for sink connectors
-    ]
-
-    TOPIC_KEYS = ["kafka.topic", "topics", "topic"]
-
-
-SUPPORTED_DATASETS = {
-    "table": ConnectorConfigKeys.TABLE_KEYS,
-    "database": ConnectorConfigKeys.DATABASE_KEYS,
-    "container_name": ConnectorConfigKeys.CONTAINER_KEYS,
+    ],
+    "database": ["database", "db.name", "snowflake.database.name"],
+    "container_name": ["s3.bucket.name"],
 }
 
 
@@ -153,41 +56,6 @@ class KafkaConnectClient:
             auth = f"{config.KafkaConnectConfig.username}:{config.KafkaConnectConfig.password.get_secret_value()}"
         self.client = KafkaConnect(url=url, auth=auth, ssl_verify=ssl_verify)
 
-        # Detect if this is Confluent Cloud (managed connectors)
-        parsed_url = urlparse(url)
-        self.is_confluent_cloud = parsed_url.hostname == "api.confluent.cloud"
-
-    def _infer_cdc_topics_from_server_name(
-        self, database_server_name: str
-    ) -> Optional[List[KafkaConnectTopics]]:
-        """
-        For CDC connectors, infer topic names based on database.server.name or topic.prefix.
-        CDC connectors create topics with pattern: {server-name}.{database}.{table}
-
-        This is a workaround for Confluent Cloud which doesn't expose topic lists.
-        We look for topics that start with the server name prefix.
-
-        Args:
-            database_server_name: The database.server.name or topic.prefix from config
-
-        Returns:
-            List of inferred KafkaConnectTopics, or None
-        """
-        if not database_server_name or not self.is_confluent_cloud:
-            return None
-
-        try:
-            # Get all connectors and check their topics
-            # Note: This is a best-effort approach for Confluent Cloud
-            # In practice, the messaging service should already have ingested these topics
-            logger.debug(
-                f"CDC connector detected with server name: {database_server_name}"
-            )
-            return None  # Topics will be matched via messaging service during lineage
-        except Exception as exc:
-            logger.debug(f"Unable to infer CDC topics: {exc}")
-            return None
-
     def _enrich_connector_details(
         self, connector_details: KafkaConnectPipelineDetails, connector_name: str
     ) -> None:
@@ -202,49 +70,10 @@ class KafkaConnectClient:
                 connector_details.config
             )
 
-            # For CDC connectors without explicit topics, try to infer from server name
-            if (
-                not connector_details.topics
-                and connector_details.conn_type.lower() == "source"
-            ):
-                database_server_name = connector_details.config.get(
-                    "database.server.name"
-                ) or connector_details.config.get("topic.prefix")
-                if database_server_name:
-                    inferred_topics = self._infer_cdc_topics_from_server_name(
-                        database_server_name
-                    )
-                    if inferred_topics:
-                        connector_details.topics = inferred_topics
-
     def get_cluster_info(self) -> Optional[dict]:
         """
         Get the version and other details of the Kafka Connect cluster.
-
-        For Confluent Cloud, the root endpoint is not supported, so we use
-        the /connectors endpoint to verify authentication and connectivity.
         """
-        if self.is_confluent_cloud:
-            # Confluent Cloud doesn't support the root endpoint (/)
-            # Use /connectors to test authentication and connectivity
-            logger.info(
-                "Confluent Cloud detected - testing connection via connectors list endpoint"
-            )
-            try:
-                connectors = self.client.list_connectors()
-                # Connection successful - return a valid response
-                logger.info(
-                    f"Confluent Cloud connection successful - found {len(connectors) if connectors else 0} connectors"
-                )
-                return {
-                    "version": "confluent-cloud",
-                    "commit": "managed",
-                    "kafka_cluster_id": "confluent-managed",
-                }
-            except Exception as exc:
-                logger.error(f"Failed to connect to Confluent Cloud: {exc}")
-                raise
-
         return self.client.get_cluster_info()
 
     def get_connectors_list(
@@ -293,97 +122,19 @@ class KafkaConnectClient:
     def get_connector_config(self, connector: str) -> Optional[dict]:
         """
         Get the details of a single connector.
-
-        For Confluent Cloud, the API returns configs as an array of {config, value} objects.
-        For self-hosted Kafka Connect, it returns a flat config dictionary.
-
         Args:
             connector (str): The name of the connector.
         """
         try:
             result = self.client.get_connector(connector=connector)
-            if not result:
-                return None
-
-            # Check if this is Confluent Cloud format (array of {config, value})
-            if self.is_confluent_cloud and "configs" in result:
-                # Transform Confluent Cloud format: [{config: "key", value: "val"}] -> {key: val}
-                configs_array = result.get("configs", [])
-                if isinstance(configs_array, list):
-                    config_dict = {
-                        item["config"]: item["value"]
-                        for item in configs_array
-                        if isinstance(item, dict)
-                        and "config" in item
-                        and "value" in item
-                    }
-                    return config_dict or None
-
-            # Standard self-hosted Kafka Connect format
-            return result.get("config")
-
+            if result:
+                return result.get("config")
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(f"Unable to get connector configuration details {exc}")
 
         return None
 
-    def extract_column_mappings(
-        self, connector_config: dict
-    ) -> Optional[List[KafkaConnectColumnMapping]]:
-        """
-        Extract column mappings from connector configuration.
-        For Debezium and JDBC connectors, columns are typically mapped 1:1
-        unless transforms are applied.
-
-        Args:
-            connector_config: The connector configuration dictionary
-
-        Returns:
-            List of KafkaConnectColumnMapping objects if mappings can be inferred
-        """
-        if not connector_config or not isinstance(connector_config, dict):
-            logger.debug("Invalid connector_config: expected dict")
-            return None
-
-        try:
-            column_mappings = []
-
-            # Check for SMT (Single Message Transform) configurations
-            transforms = connector_config.get("transforms", "")
-            if not transforms:
-                return None
-
-            transform_list = [t.strip() for t in transforms.split(",")]
-            for transform in transform_list:
-                transform_type = connector_config.get(
-                    f"transforms.{transform}.type", ""
-                )
-
-                # ReplaceField transform can rename columns
-                if "ReplaceField" in transform_type:
-                    renames = connector_config.get(
-                        f"transforms.{transform}.renames", ""
-                    )
-                    if renames:
-                        for rename in renames.split(","):
-                            if ":" in rename:
-                                source_col, target_col = rename.split(":", 1)
-                                column_mappings.append(
-                                    KafkaConnectColumnMapping(
-                                        source_column=source_col.strip(),
-                                        target_column=target_col.strip(),
-                                    )
-                                )
-
-            return column_mappings if column_mappings else None
-
-        except (KeyError, AttributeError, ValueError) as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Unable to extract column mappings: {exc}")
-
-        return None
-
     def get_connector_dataset_info(
         self, connector_config: dict
     ) -> Optional[KafkaConnectDatasetDetails]:
@@ -392,36 +143,27 @@ class KafkaConnectClient:
         Checks in the connector configurations for dataset fields
         if any related field is found returns the result
         Args:
-            connector_config: The connector configuration dictionary
+            connector (str): The name of the connector.
         Returns:
-            Optional[KafkaConnectDatasetDetails]: Dataset information including
-                table, database, or container_name if found, or None otherwise
+            Optional[Dict]: A dictionary containing dataset information
+                        (type, table, database, or bucket_name)
+                        if a dataset is found, or None if the connector
+                        is not found, has no dataset, or an error occurs.
         """
-        if not connector_config or not isinstance(connector_config, dict):
-            logger.debug("Invalid connector_config: expected dict")
-            return None
-
         try:
+            if not connector_config:
+                return None
+
             result = {}
-            for dataset_type, config_keys in SUPPORTED_DATASETS.items():
-                for key in config_keys:
+            for dataset in SUPPORTED_DATASETS or []:
+                for key in SUPPORTED_DATASETS[dataset] or []:
                     if connector_config.get(key):
-                        result[dataset_type] = connector_config[key]
-
-            # Only create dataset details if we have meaningful dataset information
-            # For CDC connectors, database.server.name/topic.prefix are captured
-            # but don't represent actual table names, so skip dataset creation
-            # We need either: table name OR container name
-            if result and (result.get("table") or result.get("container_name")):
-                dataset_details = KafkaConnectDatasetDetails(**result)
-                dataset_details.column_mappings = (
-                    self.extract_column_mappings(connector_config) or []
-                )
-                return dataset_details
+                        result[dataset] = connector_config[key]
+                        return KafkaConnectDatasetDetails(**result)
 
-        except (KeyError, ValueError, TypeError) as exc:
+        except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Unable to get connector dataset details: {exc}")
+            logger.warning(f"Unable to get connector dataset details {exc}")
 
         return None
 
@@ -431,9 +173,6 @@ class KafkaConnectClient:
         """
         Get the list of topics for a connector.
 
-        For Confluent Cloud, the /topics endpoint is not supported, so we extract
-        topics from the connector configuration instead.
-
         Args:
             connector (str): The name of the connector.
 
@@ -444,42 +183,15 @@ class KafkaConnectClient:
                                             or an error occurs.
         """
         try:
-            if self.is_confluent_cloud:
-                # Confluent Cloud doesn't support /connectors/{name}/topics endpoint
-                # Extract topics from connector config instead
-                config = self.get_connector_config(connector=connector)
-                if config:
-                    topics = []
-                    # Check common topic configuration keys
-                    for key in ConnectorConfigKeys.TOPIC_KEYS:
-                        if key in config:
-                            topic_value = config[key]
-                            # Handle single topic or comma-separated list
-                            if isinstance(topic_value, str):
-                                topic_list = [t.strip() for t in topic_value.split(",")]
-                                topics.extend(
-                                    [
-                                        KafkaConnectTopics(name=topic)
-                                        for topic in topic_list
-                                    ]
-                                )
-
-                    if topics:
-                        logger.info(
-                            f"Extracted {len(topics)} topics from Confluent Cloud connector config"
-                        )
-                        return topics
-            else:
-                # Self-hosted Kafka Connect supports /topics endpoint
-                result = self.client.list_connector_topics(connector=connector).get(
-                    connector
-                )
-                if result:
-                    topics = [
-                        KafkaConnectTopics(name=topic)
-                        for topic in result.get("topics") or []
-                    ]
-                    return topics
+            result = self.client.list_connector_topics(connector=connector).get(
+                connector
+            )
+            if result:
+                topics = [
+                    KafkaConnectTopics(name=topic)
+                    for topic in result.get("topics") or []
+                ]
+                return topics
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Unable to get connector Topics {exc}")