cledar-sdk 2.0.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

cledar/kafka/config/schemas.py

@@ -0,0 +1,262 @@
+"""Configuration schemas for Kafka clients."""
+
+from enum import StrEnum
+
+from pydantic import field_validator
+from pydantic.dataclasses import dataclass
+
+
+class KafkaSecurityProtocol(StrEnum):
+    """Supported Kafka security protocols."""
+
+    PLAINTEXT = "PLAINTEXT"
+    SSL = "SSL"
+    SASL_PLAINTEXT = "SASL_PLAINTEXT"
+    SASL_SSL = "SASL_SSL"
+
+
+class KafkaSaslMechanism(StrEnum):
+    """Supported Kafka SASL mechanisms."""
+
+    PLAIN = "PLAIN"
+    SCRAM_SHA_256 = "SCRAM-SHA-256"
+    SCRAM_SHA_512 = "SCRAM-SHA-512"
+    GSSAPI = "GSSAPI"
+
+
+def _validate_kafka_servers(v: list[str] | str) -> list[str] | str:
+    """Validate kafka_servers is not empty.
+
+    Args:
+        v: List of Kafka broker addresses or a comma-separated string.
+
+    Returns:
+        list[str] | str: The validated value.
+
+    Raises:
+        ValueError: If the value is empty.
+
+    """
+    if isinstance(v, str) and v.strip() == "":
+        raise ValueError("kafka_servers cannot be empty")
+    if isinstance(v, list) and len(v) == 0:
+        raise ValueError("kafka_servers cannot be empty list")
+    return v
+
+
+def _validate_non_negative(v: int) -> int:
+    """Validate that timeout values are non-negative.
+
+    Args:
+        v: Timeout value to validate.
+
+    Returns:
+        int: The validated value.
+
+    Raises:
+        ValueError: If the value is negative.
+
+    """
+    if v < 0:
+        raise ValueError("timeout values must be non-negative")
+    return v
+
+
+@dataclass(frozen=True)
+class KafkaProducerConfig:
+    """Configuration for Kafka Producer.
+
+    Args:
+        kafka_servers: List of Kafka broker addresses or comma-separated string
+        kafka_group_id: Consumer group identifier
+        kafka_topic_prefix: Optional prefix for topic names
+        kafka_block_buffer_time_sec: Time to block when buffer is full
+        kafka_connection_check_timeout_sec: Timeout for connection health checks
+        kafka_connection_check_interval_sec: Interval between connection checks
+        kafka_partitioner: Partitioning strategy for messages
+        compression_type: Compression type for messages (gzip, snappy, lz4, zstd,
+            or None)
+
+    """
+
+    kafka_servers: list[str] | str
+    kafka_group_id: str
+    kafka_security_protocol: KafkaSecurityProtocol | None = None
+    kafka_sasl_mechanism: KafkaSaslMechanism | None = None
+    kafka_sasl_username: str | None = None
+    kafka_sasl_password: str | None = None
+    kafka_topic_prefix: str | None = None
+    kafka_block_buffer_time_sec: int = 10
+    kafka_connection_check_timeout_sec: int = 5
+    kafka_connection_check_interval_sec: int = 60
+    kafka_partitioner: str = "consistent_random"
+    compression_type: str | None = "gzip"
+
+    @field_validator("kafka_servers")
+    @classmethod
+    def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        """Validate kafka_servers field.
+
+        Args:
+            v: List of Kafka broker addresses or a comma-separated string.
+
+        Returns:
+            list[str] | str: The validated value.
+
+        """
+        return _validate_kafka_servers(v)
+
+    @field_validator(
+        "kafka_block_buffer_time_sec",
+        "kafka_connection_check_timeout_sec",
+        "kafka_connection_check_interval_sec",
+    )
+    @classmethod
+    def validate_positive_timeouts(cls, v: int) -> int:
+        """Validate positive timeout values.
+
+        Args:
+            v: Timeout value to validate.
+
+        Returns:
+            int: The validated value.
+
+        """
+        return _validate_non_negative(v)
+
+    def to_kafka_config(self) -> dict[str, list[str] | str | None]:
+        """Build Kafka producer configuration dictionary.
+
+        Returns:
+            dict[str, list[str] | str | None]: Kafka client configuration.
+
+        """
+        config = {
+            "bootstrap.servers": self.kafka_servers,
+            "client.id": self.kafka_group_id,
+            "compression.type": self.compression_type,
+            "partitioner": self.kafka_partitioner,
+        }
+
+        # Add SASL configuration if specified
+        if self.kafka_security_protocol:
+            config["security.protocol"] = self.kafka_security_protocol.value
+            if self.kafka_sasl_mechanism:
+                config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
+            if self.kafka_sasl_username:
+                config["sasl.username"] = self.kafka_sasl_username
+            if self.kafka_sasl_password:
+                config["sasl.password"] = self.kafka_sasl_password
+
+        return config
+
+
+@dataclass(frozen=True)
+class KafkaConsumerConfig:
+    """Configuration for Kafka Consumer.
+
+    Args:
+        kafka_servers: List of Kafka broker addresses or comma-separated string
+        kafka_group_id: Consumer group identifier
+        kafka_offset: Starting offset position ('earliest', 'latest', or specific
+            offset)
+        kafka_topic_prefix: Optional prefix for topic names
+        kafka_block_consumer_time_sec: Time to block waiting for messages
+        kafka_connection_check_timeout_sec: Timeout for connection health checks
+        kafka_auto_commit_interval_ms: Interval for automatic offset commits
+        kafka_connection_check_interval_sec: Interval between connection checks
+
+    """
+
+    kafka_servers: list[str] | str
+    kafka_group_id: str
+    kafka_security_protocol: KafkaSecurityProtocol | None = None
+    kafka_sasl_mechanism: KafkaSaslMechanism | None = None
+    kafka_sasl_username: str | None = None
+    kafka_sasl_password: str | None = None
+    kafka_offset: str = "latest"
+    kafka_topic_prefix: str | None = None
+    kafka_block_consumer_time_sec: int = 2
+    kafka_connection_check_timeout_sec: int = 5
+    kafka_auto_commit_interval_ms: int = 1000
+    kafka_connection_check_interval_sec: int = 60
+
+    @field_validator("kafka_servers")
+    @classmethod
+    def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        """Validate kafka_servers field.
+
+        Args:
+            v: List of Kafka broker addresses or a comma-separated string.
+
+        Returns:
+            list[str] | str: The validated value.
+
+        """
+        return _validate_kafka_servers(v)
+
+    @field_validator("kafka_offset")
+    @classmethod
+    def validate_kafka_offset(cls, v: str) -> str:
+        """Validate kafka_offset field.
+
+        Args:
+            v: Offset value to validate.
+
+        Returns:
+            str: The validated offset value.
+
+        Raises:
+            ValueError: If the value is empty.
+
+        """
+        if v.strip() == "":
+            raise ValueError("kafka_offset cannot be empty")
+        return v
+
+    @field_validator(
+        "kafka_block_consumer_time_sec",
+        "kafka_connection_check_timeout_sec",
+        "kafka_auto_commit_interval_ms",
+        "kafka_connection_check_interval_sec",
+    )
+    @classmethod
+    def validate_positive_timeouts(cls, v: int) -> int:
+        """Validate positive timeout values.
+
+        Args:
+            v: Timeout value to validate.
+
+        Returns:
+            int: The validated value.
+
+        """
+        return _validate_non_negative(v)
+
+    def to_kafka_config(self) -> dict[str, int | list[str] | str]:
+        """Build Kafka consumer configuration dictionary.
+
+        Returns:
+            dict[str, int | list[str] | str]: Kafka client configuration.
+
+        """
+        config = {
+            "bootstrap.servers": self.kafka_servers,
+            "enable.auto.commit": False,
+            "enable.partition.eof": False,
+            "auto.commit.interval.ms": self.kafka_auto_commit_interval_ms,
+            "auto.offset.reset": self.kafka_offset,
+            "group.id": self.kafka_group_id,
+        }
+
+        # Add SASL configuration if specified
+        if self.kafka_security_protocol:
+            config["security.protocol"] = self.kafka_security_protocol.value
+            if self.kafka_sasl_mechanism:
+                config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
+            if self.kafka_sasl_username:
+                config["sasl.username"] = self.kafka_sasl_username
+            if self.kafka_sasl_password:
+                config["sasl.password"] = self.kafka_sasl_password
+
+        return config

cledar/kafka/exceptions.py

@@ -0,0 +1,17 @@
+"""Kafka-related exceptions for the Cledar SDK."""
+
+
+class KafkaProducerNotConnectedError(Exception):
+    """Custom exception for KafkaProducer to indicate it is not connected."""
+
+
+class KafkaConsumerNotConnectedError(Exception):
+    """Custom exception for KafkaConsumer to indicate it is not connected."""
+
+
+class KafkaConnectionError(Exception):
+    """Custom exception to indicate connection failures."""
+
+
+class KafkaConsumerError(Exception):
+    """Custom exception for KafkaConsumer to indicate errors."""

cledar/kafka/handlers/dead_letter.py

@@ -0,0 +1,88 @@
+"""Dead letter handler for Kafka messages."""
+
+import json
+
+from ..clients.producer import KafkaProducer
+from ..logger import logger
+from ..models.message import KafkaMessage
+from ..models.output import FailedMessageData
+
+
+class DeadLetterHandler:
+    """A handler for handling failed messages and sending them to a DLQ topic."""
+
+    def __init__(self, producer: KafkaProducer, dlq_topic: str) -> None:
+        """Initialize DeadLetterHandler with a Kafka producer and DLQ topic.
+
+        Args:
+            producer: KafkaProducer instance.
+            dlq_topic: The name of the DLQ Kafka topic.
+
+        """
+        self.producer: KafkaProducer = producer
+        self.dlq_topic: str = dlq_topic
+
+    def handle(
+        self,
+        message: KafkaMessage,
+        failures_details: list[FailedMessageData] | None,
+    ) -> None:
+        """Handle a failed message by sending it to the DLQ topic.
+
+        Args:
+            message: The original Kafka message.
+            failures_details: A list of FailedMessageData.
+
+        """
+        logger.info("Handling message for DLQ.")
+
+        kafka_headers = self._build_headers(failures_details=failures_details)
+
+        logger.info("DLQ message built successfully.")
+        self._send_message(message.value, message.key, kafka_headers)
+
+    def _build_headers(
+        self,
+        failures_details: list[FailedMessageData] | None,
+    ) -> list[tuple[str, bytes]]:
+        """Build Kafka headers containing exception details.
+
+        Args:
+            failures_details: A list of FailedMessageData.
+
+        Returns:
+            list[tuple[str, bytes]]: A list of Kafka headers.
+
+        """
+        headers: list[tuple[str, bytes]] = []
+
+        if failures_details:
+            failures_json = json.dumps(
+                [failure.model_dump() for failure in failures_details]
+            )
+            headers.append(("failures_details", failures_json.encode("utf-8")))
+
+        return headers
+
+    def _send_message(
+        self,
+        message: str | None,
+        key: str | None,
+        headers: list[tuple[str, bytes]],
+    ) -> None:
+        """Send a DLQ message to the Kafka DLQ topic with headers.
+
+        Args:
+            message: The DLQ message payload.
+            key: The original Kafka message key.
+            headers: Kafka headers containing exception details.
+
+        """
+        self.producer.send(
+            topic=self.dlq_topic, value=message, key=key, headers=headers
+        )
+        logger.info(
+            "Message sent to DLQ topic successfully with key: %s and headers: %s",
+            key,
+            headers,
+        )

cledar/kafka/handlers/parser.py

@@ -0,0 +1,83 @@
+"""Kafka message parser module."""
+
+import json
+
+from pydantic import BaseModel
+
+from ..models.input import (
+    InputKafkaMessage,
+)
+from ..models.message import KafkaMessage
+
+
+class IncorrectMessageValueError(Exception):
+    """Message needs to have `value` field present in order to be parsed.
+
+    This is unless `model` is set to `None`.
+    """
+
+
+class InputParser[Payload: BaseModel]:
+    """Parser for Kafka messages into Pydantic models.
+
+    Generic class for parsing Kafka messages.
+    """
+
+    def __init__(self, model: type[Payload]) -> None:
+        """Initialize InputParser with a Pydantic model.
+
+        Args:
+            model: The Pydantic model to validate messages against.
+
+        """
+        self.model: type[Payload] = model
+
+    def parse_json(self, json_str: str) -> Payload:
+        """Parse JSON text and validate into the target Payload model.
+
+        Invalid JSON should raise IncorrectMessageValueError, while schema
+        validation errors should bubble up as ValidationError.
+
+        Args:
+            json_str: The JSON string to parse.
+
+        Returns:
+            Payload: The validated Pydantic model instance.
+
+        Raises:
+            IncorrectMessageValueError: If the JSON is invalid.
+
+        """
+        try:
+            data = json.loads(json_str)
+        except json.JSONDecodeError as exc:
+            # Normalize invalid JSON into our domain-specific error
+            raise IncorrectMessageValueError from exc
+        return self.model.model_validate(data)
+
+    def parse_message(self, message: KafkaMessage) -> InputKafkaMessage[Payload]:
+        """Parse a Kafka message into an InputKafkaMessage with a validated payload.
+
+        Args:
+            message: The Kafka message to parse.
+
+        Returns:
+            InputKafkaMessage[Payload]: The parsed message with payload.
+
+        Raises:
+            IncorrectMessageValueError: If message value is missing but required.
+
+        """
+        if message.value is None and self.model is not None:
+            raise IncorrectMessageValueError
+
+        obj = self.parse_json(message.value)
+
+        return InputKafkaMessage(
+            key=message.key,
+            value=message.value,
+            payload=obj,
+            topic=message.topic,
+            offset=message.offset,
+            partition=message.partition,
+        )

cledar/kafka/logger.py

@@ -0,0 +1,5 @@
+"""Logger configuration for the Kafka module."""
+
+import logging
+
+logger = logging.getLogger("kafka_service")

cledar/kafka/models/input.py

@@ -0,0 +1,17 @@
+"""Input Kafka message model."""
+
+import dataclasses
+from typing import TypeVar
+
+from pydantic import BaseModel
+
+from .message import KafkaMessage
+
+Payload = TypeVar("Payload", bound=BaseModel)
+
+
+@dataclasses.dataclass
+class InputKafkaMessage[Payload](KafkaMessage):
+    """Kafka message with a parsed and validated Pydantic payload."""
+
+    payload: Payload

cledar/kafka/models/message.py

@@ -0,0 +1,14 @@
+"""Kafka message data class."""
+
+from pydantic.dataclasses import dataclass
+
+
+@dataclass
+class KafkaMessage:
+    """Base Kafka message representation."""
+
+    topic: str
+    value: str | None
+    key: str | None
+    offset: int | None
+    partition: int | None

cledar/kafka/models/output.py

@@ -0,0 +1,12 @@
+"""Output models for Kafka module."""
+
+import pydantic
+
+
+class FailedMessageData(pydantic.BaseModel):
+    """Data structure for recording message processing failures."""
+
+    raised_at: str
+    exception_message: str | None
+    exception_trace: str | None
+    failure_reason: str | None

cledar/kafka/tests/.env.test.kafka

@@ -0,0 +1,3 @@
+KAFKA_SERVERS="localhost:9092"
+KAFKA_GROUP_ID="test-group"
+KAFKA_TOPIC_PREFIX="test."