cledar-sdk 2.0.3__py3-none-any.whl → 2.1.1__py3-none-any.whl

cledar/__init__.py

@@ -0,0 +1 @@
+"""Cledar Python SDK for data platform services and integrations."""

cledar/kafka/__init__.py

@@ -1,3 +1,5 @@
+"""Kafka client and utilities for Cledar SDK."""
+
 from .clients.base import BaseKafkaClient
 from .clients.consumer import KafkaConsumer
 from .clients.producer import KafkaProducer

cledar/kafka/clients/base.py

@@ -1,3 +1,5 @@
+"""Base Kafka client module."""
+
 import threading
 
 from confluent_kafka import Consumer, KafkaException, Producer
@@ -15,11 +17,18 @@ from ..logger import logger
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class BaseKafkaClient:
+    """Base class for Kafka clients.
+
+    This class provides common functionality for both producers and consumers,
+    such as connection monitoring and shutdown handling.
+    """
+
     config: KafkaProducerConfig | KafkaConsumerConfig
     client: Producer | Consumer | None = None
     connection_check_thread: threading.Thread | None = None
 
     def __post_init__(self) -> None:
+        """Initialize the client with instance-level events."""
         # Create instance-level stop event
         self._stop_event = threading.Event()
         logger.info(
@@ -27,6 +36,7 @@ class BaseKafkaClient:
         )
 
     def start_connection_check_thread(self) -> None:
+        """Start a background thread to monitor the Kafka connection."""
         if self.connection_check_thread is None:
             self.connection_check_thread = threading.Thread(
                 target=self._monitor_connection
@@ -48,6 +58,12 @@ class BaseKafkaClient:
                 logger.exception(f"{self.__class__.__name__} connection check failed.")
 
     def is_alive(self) -> bool:
+        """Check if the client is currently connected and alive.
+
+        Returns:
+            bool: True if connected, False otherwise.
+
+        """
         try:
             self.check_connection()
             return True
@@ -59,11 +75,13 @@ class BaseKafkaClient:
             return False
 
     def check_connection(self) -> None:
-        """
-        when the broker is not available (or the address is wrong)
-        the 'connection refused' error is not caught
-        https://github.com/confluentinc/confluent-kafka-python/issues/941
-        the below is far-from-perfect workaround handling that
+        """Check the connection to Kafka servers.
+
+        When the broker is not available (or the address is wrong),
+        the 'connection refused' error is not caught.
+        Ref: https://github.com/confluentinc/confluent-kafka-python/issues/941
+        The below is far-from-perfect workaround handling that.
+
         """
         if self.client is None:
             logger.error(
@@ -83,6 +101,7 @@ class BaseKafkaClient:
             raise KafkaConnectionError from exception
 
     def shutdown(self) -> None:
+        """Shutdown the client and stop connection monitoring."""
         logger.info("Closing %s...", self.__class__.__name__)
         self._stop_event.set()
         if self.connection_check_thread is not None:

cledar/kafka/clients/consumer.py

@@ -1,3 +1,5 @@
+"""Kafka consumer client module."""
+
 from confluent_kafka import Consumer, KafkaException
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -16,10 +18,17 @@ from .base import BaseKafkaClient
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class KafkaConsumer(BaseKafkaClient):
+    """Kafka consumer client.
+
+    This class provides methods to connect to Kafka, subscribe to topics,
+    and consume messages.
+    """
+
     config: KafkaConsumerConfig
     client: Consumer | None = None
 
     def connect(self) -> None:
+        """Connect to Kafka servers and start connection monitoring."""
         self.client = Consumer(self.config.to_kafka_config())
         self.check_connection()
         logger.info(
@@ -29,6 +38,12 @@ class KafkaConsumer(BaseKafkaClient):
         self.start_connection_check_thread()
 
     def subscribe(self, topics: list[str]) -> None:
+        """Subscribe to a list of topics.
+
+        Args:
+            topics: A list of topic names to subscribe to.
+
+        """
         if self.client is None:
             logger.error(
                 consumer_not_connected_msg,
@@ -56,6 +71,13 @@ class KafkaConsumer(BaseKafkaClient):
             raise exception
 
     def consume_next(self) -> KafkaMessage | None:
+        """Consume the next message from subscribed topics.
+
+        Returns:
+            KafkaMessage | None: The consumed message or None if no message is
+                available.
+
+        """
         if self.client is None:
             logger.error(consumer_not_connected_msg)
             raise KafkaConsumerNotConnectedError
@@ -94,6 +116,12 @@ class KafkaConsumer(BaseKafkaClient):
             raise exception
 
     def commit(self, message: KafkaMessage) -> None:
+        """Commit offsets for the current message.
+
+        Args:
+            message: The message for which to commit offsets.
+
+        """
         if self.client is None:
             logger.error(consumer_not_connected_msg)
             raise KafkaConsumerNotConnectedError

cledar/kafka/clients/producer.py

@@ -1,3 +1,5 @@
+"""Kafka producer client module."""
+
 from confluent_kafka import KafkaException, Producer
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
@@ -13,10 +15,16 @@ from .base import BaseKafkaClient
 
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class KafkaProducer(BaseKafkaClient):
+    """Kafka producer client.
+
+    This class provides methods to connect to Kafka and send messages.
+    """
+
     config: KafkaProducerConfig
     client: Producer | None = None
 
     def connect(self) -> None:
+        """Connect to Kafka servers and start connection monitoring."""
         self.client = Producer(self.config.to_kafka_config())
         self.check_connection()
         logger.info(
@@ -32,6 +40,15 @@ class KafkaProducer(BaseKafkaClient):
         key: str | None,
         headers: list[tuple[str, bytes]] | None = None,
     ) -> None:
+        """Send a message to a Kafka topic.
+
+        Args:
+            topic: The name of the topic to send the message to.
+            value: The message value.
+            key: The message key.
+            headers: Optional list of message headers.
+
+        """
         if self.client is None:
             logger.error(
                 "KafkaProducer is not connected. Call 'connect' first.",

cledar/kafka/config/schemas.py

@@ -1,10 +1,12 @@
-from enum import Enum
+"""Configuration schemas for Kafka clients."""
+
+from enum import StrEnum
 
 from pydantic import field_validator
 from pydantic.dataclasses import dataclass
 
 
-class KafkaSecurityProtocol(str, Enum):
+class KafkaSecurityProtocol(StrEnum):
     """Supported Kafka security protocols."""
 
     PLAINTEXT = "PLAINTEXT"
@@ -13,7 +15,7 @@ class KafkaSecurityProtocol(str, Enum):
     SASL_SSL = "SASL_SSL"
 
 
-class KafkaSaslMechanism(str, Enum):
+class KafkaSaslMechanism(StrEnum):
     """Supported Kafka SASL mechanisms."""
 
     PLAIN = "PLAIN"
@@ -23,7 +25,18 @@ class KafkaSaslMechanism(str, Enum):
 
 
 def _validate_kafka_servers(v: list[str] | str) -> list[str] | str:
-    """Validate kafka_servers is not empty."""
+    """Validate kafka_servers is not empty.
+
+    Args:
+        v: List of Kafka broker addresses or a comma-separated string.
+
+    Returns:
+        list[str] | str: The validated value.
+
+    Raises:
+        ValueError: If the value is empty.
+
+    """
     if isinstance(v, str) and v.strip() == "":
         raise ValueError("kafka_servers cannot be empty")
     if isinstance(v, list) and len(v) == 0:
@@ -32,7 +45,18 @@ def _validate_kafka_servers(v: list[str] | str) -> list[str] | str:
 
 
 def _validate_non_negative(v: int) -> int:
-    """Validate that timeout values are non-negative."""
+    """Validate that timeout values are non-negative.
+
+    Args:
+        v: Timeout value to validate.
+
+    Returns:
+        int: The validated value.
+
+    Raises:
+        ValueError: If the value is negative.
+
+    """
     if v < 0:
         raise ValueError("timeout values must be non-negative")
     return v
@@ -52,6 +76,7 @@ class KafkaProducerConfig:
         kafka_partitioner: Partitioning strategy for messages
         compression_type: Compression type for messages (gzip, snappy, lz4, zstd,
             or None)
+
     """
 
     kafka_servers: list[str] | str
@@ -70,6 +95,15 @@ class KafkaProducerConfig:
     @field_validator("kafka_servers")
     @classmethod
     def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        """Validate kafka_servers field.
+
+        Args:
+            v: List of Kafka broker addresses or a comma-separated string.
+
+        Returns:
+            list[str] | str: The validated value.
+
+        """
         return _validate_kafka_servers(v)
 
     @field_validator(
@@ -79,10 +113,24 @@ class KafkaProducerConfig:
     )
     @classmethod
     def validate_positive_timeouts(cls, v: int) -> int:
+        """Validate positive timeout values.
+
+        Args:
+            v: Timeout value to validate.
+
+        Returns:
+            int: The validated value.
+
+        """
         return _validate_non_negative(v)
 
     def to_kafka_config(self) -> dict[str, list[str] | str | None]:
-        """Build Kafka producer configuration dictionary."""
+        """Build Kafka producer configuration dictionary.
+
+        Returns:
+            dict[str, list[str] | str | None]: Kafka client configuration.
+
+        """
         config = {
             "bootstrap.servers": self.kafka_servers,
             "client.id": self.kafka_group_id,
@@ -117,6 +165,7 @@ class KafkaConsumerConfig:
         kafka_connection_check_timeout_sec: Timeout for connection health checks
         kafka_auto_commit_interval_ms: Interval for automatic offset commits
         kafka_connection_check_interval_sec: Interval between connection checks
+
     """
 
     kafka_servers: list[str] | str
@@ -135,11 +184,32 @@ class KafkaConsumerConfig:
     @field_validator("kafka_servers")
     @classmethod
     def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        """Validate kafka_servers field.
+
+        Args:
+            v: List of Kafka broker addresses or a comma-separated string.
+
+        Returns:
+            list[str] | str: The validated value.
+
+        """
         return _validate_kafka_servers(v)
 
     @field_validator("kafka_offset")
     @classmethod
     def validate_kafka_offset(cls, v: str) -> str:
+        """Validate kafka_offset field.
+
+        Args:
+            v: Offset value to validate.
+
+        Returns:
+            str: The validated offset value.
+
+        Raises:
+            ValueError: If the value is empty.
+
+        """
         if v.strip() == "":
             raise ValueError("kafka_offset cannot be empty")
         return v
@@ -152,10 +222,24 @@ class KafkaConsumerConfig:
     )
     @classmethod
     def validate_positive_timeouts(cls, v: int) -> int:
+        """Validate positive timeout values.
+
+        Args:
+            v: Timeout value to validate.
+
+        Returns:
+            int: The validated value.
+
+        """
         return _validate_non_negative(v)
 
     def to_kafka_config(self) -> dict[str, int | list[str] | str]:
-        """Build Kafka consumer configuration dictionary."""
+        """Build Kafka consumer configuration dictionary.
+
+        Returns:
+            dict[str, int | list[str] | str]: Kafka client configuration.
+
+        """
         config = {
             "bootstrap.servers": self.kafka_servers,
             "enable.auto.commit": False,

cledar/kafka/exceptions.py

@@ -1,22 +1,17 @@
+"""Kafka-related exceptions for the Cledar SDK."""
+
+
 class KafkaProducerNotConnectedError(Exception):
-    """
-    Custom exception for KafkaProducer to indicate it is not connected.
-    """
+    """Custom exception for KafkaProducer to indicate it is not connected."""
 
 
 class KafkaConsumerNotConnectedError(Exception):
-    """
-    Custom exception for KafkaConsumer to indicate it is not connected.
-    """
+    """Custom exception for KafkaConsumer to indicate it is not connected."""
 
 
 class KafkaConnectionError(Exception):
-    """
-    Custom exception to indicate connection failures.
-    """
+    """Custom exception to indicate connection failures."""
 
 
 class KafkaConsumerError(Exception):
-    """
-    Custom exception for KafkaConsumer to indicate errors.
-    """
+    """Custom exception for KafkaConsumer to indicate errors."""

cledar/kafka/handlers/dead_letter.py

@@ -1,3 +1,5 @@
+"""Dead letter handler for Kafka messages."""
+
 import json
 
 from ..clients.producer import KafkaProducer
@@ -7,16 +9,15 @@ from ..models.output import FailedMessageData
 
 
 class DeadLetterHandler:
-    """
-    A Handler for handling failed messages and sending them to a DLQ topic.
-    """
+    """A handler for handling failed messages and sending them to a DLQ topic."""
 
     def __init__(self, producer: KafkaProducer, dlq_topic: str) -> None:
-        """
-        Initialize DeadLetterHandler with a Kafka producer and DLQ topic.
+        """Initialize DeadLetterHandler with a Kafka producer and DLQ topic.
+
+        Args:
+            producer: KafkaProducer instance.
+            dlq_topic: The name of the DLQ Kafka topic.
 
-        :param producer: KafkaProducer instance.
-        :param dlq_topic: The name of the DLQ Kafka topic.
         """
         self.producer: KafkaProducer = producer
         self.dlq_topic: str = dlq_topic
@@ -26,11 +27,12 @@ class DeadLetterHandler:
         message: KafkaMessage,
         failures_details: list[FailedMessageData] | None,
     ) -> None:
-        """
-        Handles a failed message by sending it to the DLQ topic.
+        """Handle a failed message by sending it to the DLQ topic.
+
+        Args:
+            message: The original Kafka message.
+            failures_details: A list of FailedMessageData.
 
-        :param message: The original Kafka message.
-        :param failures_details: A list of FailedMessageData.
         """
         logger.info("Handling message for DLQ.")
 
@@ -43,11 +45,14 @@ class DeadLetterHandler:
         self,
         failures_details: list[FailedMessageData] | None,
     ) -> list[tuple[str, bytes]]:
-        """
-        Builds Kafka headers containing exception details.
+        """Build Kafka headers containing exception details.
+
+        Args:
+            failures_details: A list of FailedMessageData.
+
+        Returns:
+            list[tuple[str, bytes]]: A list of Kafka headers.
 
-        :param failures_details: A list of FailedMessageData.
-        :return: A list of Kafka headers.
         """
         headers: list[tuple[str, bytes]] = []
 
@@ -65,12 +70,13 @@ class DeadLetterHandler:
         key: str | None,
         headers: list[tuple[str, bytes]],
     ) -> None:
-        """
-        Sends a DLQ message to the Kafka DLQ topic with headers.
+        """Send a DLQ message to the Kafka DLQ topic with headers.
+
+        Args:
+            message: The DLQ message payload.
+            key: The original Kafka message key.
+            headers: Kafka headers containing exception details.
 
-        :param message: The DLQ message payload.
-        :param key: The original Kafka message key.
-        :param headers: Kafka headers containing exception details.
         """
         self.producer.send(
             topic=self.dlq_topic, value=message, key=key, headers=headers

cledar/kafka/handlers/parser.py

@@ -1,3 +1,5 @@
+"""Kafka message parser module."""
+
 import json
 
 from pydantic import BaseModel
@@ -9,15 +11,25 @@ from ..models.message import KafkaMessage
 
 
 class IncorrectMessageValueError(Exception):
-    """
-    Message needs to have `value` field present in order to be parsed.
+    """Message needs to have `value` field present in order to be parsed.
 
     This is unless `model` is set to `None`.
     """
 
 
 class InputParser[Payload: BaseModel]:
+    """Parser for Kafka messages into Pydantic models.
+
+    Generic class for parsing Kafka messages.
+    """
+
     def __init__(self, model: type[Payload]) -> None:
+        """Initialize InputParser with a Pydantic model.
+
+        Args:
+            model: The Pydantic model to validate messages against.
+
+        """
         self.model: type[Payload] = model
 
     def parse_json(self, json_str: str) -> Payload:
@@ -25,6 +37,16 @@ class InputParser[Payload: BaseModel]:
 
         Invalid JSON should raise IncorrectMessageValueError, while schema
         validation errors should bubble up as ValidationError.
+
+        Args:
+            json_str: The JSON string to parse.
+
+        Returns:
+            Payload: The validated Pydantic model instance.
+
+        Raises:
+            IncorrectMessageValueError: If the JSON is invalid.
+
         """
         try:
             data = json.loads(json_str)
@@ -34,6 +56,18 @@ class InputParser[Payload: BaseModel]:
         return self.model.model_validate(data)
 
     def parse_message(self, message: KafkaMessage) -> InputKafkaMessage[Payload]:
+        """Parse a Kafka message into an InputKafkaMessage with a validated payload.
+
+        Args:
+            message: The Kafka message to parse.
+
+        Returns:
+            InputKafkaMessage[Payload]: The parsed message with payload.
+
+        Raises:
+            IncorrectMessageValueError: If message value is missing but required.
+
+        """
         if message.value is None and self.model is not None:
             raise IncorrectMessageValueError
 

cledar/kafka/logger.py

@@ -1,3 +1,5 @@
+"""Logger configuration for the Kafka module."""
+
 import logging
 
 logger = logging.getLogger("kafka_service")

cledar/kafka/models/input.py

@@ -1,3 +1,5 @@
+"""Input Kafka message model."""
+
 import dataclasses
 from typing import TypeVar
 
@@ -10,4 +12,6 @@ Payload = TypeVar("Payload", bound=BaseModel)
 
 @dataclasses.dataclass
 class InputKafkaMessage[Payload](KafkaMessage):
+    """Kafka message with a parsed and validated Pydantic payload."""
+
     payload: Payload

cledar/kafka/models/message.py

@@ -1,8 +1,12 @@
+"""Kafka message data class."""
+
 from pydantic.dataclasses import dataclass
 
 
 @dataclass
 class KafkaMessage:
+    """Base Kafka message representation."""
+
     topic: str
     value: str | None
     key: str | None

cledar/kafka/models/output.py

@@ -1,7 +1,11 @@
+"""Output models for Kafka module."""
+
 import pydantic
 
 
 class FailedMessageData(pydantic.BaseModel):
+    """Data structure for recording message processing failures."""
+
     raised_at: str
     exception_message: str | None
     exception_trace: str | None

cledar/kafka/utils/callbacks.py

@@ -1,9 +1,18 @@
+"""Kafka callback utilities."""
+
 from confluent_kafka import KafkaError, Message
 
 from ..logger import logger
 
 
 def delivery_callback(error: KafkaError, msg: Message) -> None:
+    """Handle message delivery reports from Kafka producer.
+
+    Args:
+        error: The delivery error (if any).
+        msg: The message that was delivered (or failed).
+
+    """
     try:
         if msg is None:
             logger.warning("Callback received a None message.")

cledar/kafka/utils/messages.py

@@ -1,3 +1,5 @@
+"""Kafka message utilities."""
+
 import json
 
 from ..logger import logger
@@ -7,6 +9,15 @@ _ID_FIELD_KEY = "id"
 
 
 def extract_id_from_value(value: str | None) -> str:
+    """Extract an ID field from a JSON-formatted message value.
+
+    Args:
+        value: The message value string.
+
+    Returns:
+        str: The extracted ID or a placeholder if not found or invalid.
+
+    """
     msg_id: str = _UNKNOWN_ID_PLACEHOLDER
     if value is None:
         return msg_id

cledar/kafka/utils/topics.py

@@ -1,2 +1,15 @@
+"""Kafka topic utilities."""
+
+
 def build_topic(topic_name: str, prefix: str | None) -> str:
+    """Build a topic name by optionally prepending a prefix.
+
+    Args:
+        topic_name: The base topic name.
+        prefix: An optional prefix to prepend.
+
+    Returns:
+        str: The full topic name.
+
+    """
     return prefix + topic_name if prefix else topic_name

cledar/kserve/__init__.py

@@ -1,3 +1,5 @@
+"""KServe integration utilities."""
+
 from .utils import get_input_topic
 
 __all__ = ["get_input_topic"]

cledar/kserve/utils.py

@@ -1,3 +1,5 @@
+"""Utilities for KServe integration and CloudEvents processing."""
+
 CE_SOURCE_HEADER = "ce-source"
 
 
@@ -18,6 +20,7 @@ def get_input_topic(headers: dict[str, str]) -> str | None:
         >>> headers = {"ce-source": "kafka://cluster#my-topic"}
         >>> get_input_topic(headers)
         'my-topic'
+
     """
     source = headers.get(CE_SOURCE_HEADER)
     if not source or "#" not in source:

cledar/logging/__init__.py

@@ -1,3 +1,5 @@
+"""Logging module with custom formatters for structured logging."""
+
 from .universal_plaintext_formatter import UniversalPlaintextFormatter
 
 __all__ = ["UniversalPlaintextFormatter"]