cledar-sdk 2.0.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

cledar/__init__.py

@@ -0,0 +1 @@
+"""Cledar Python SDK for data platform services and integrations."""

cledar/kafka/README.md

@@ -0,0 +1,239 @@
+# Kafka Service
+
+## Purpose
+
+The `cledar.kafka` package provides typed, testable wrappers around Kafka producer and consumer clients (Confluent Kafka), together with configuration schemas, message models, parsing and dead-letter handling utilities. It is designed for clarity, reliability, and easy testing (unit and integration).
+
+### Key Features
+
+- **Typed Producer/Consumer**: Simple OO wrappers for Confluent Kafka
+- **Pydantic Configs**: Validated, frozen dataclasses for producer/consumer configuration
+- **Dead Letter Handling**: Helper to route failed messages to DLQ topics
+- **Message Models**: Structured input/output models
+- **Parsing Utilities**: Safe message parsing to typed payloads
+- **Testability**: Comprehensive unit tests and Docker-based integration tests using testcontainers
+
+## Installation
+
+This package is part of the Cledar SDK. Install it using:
+
+```bash
+# Install with uv (recommended)
+uv sync --all-groups
+
+# Or with pip
+pip install -e .
+```
+
+## Usage Examples
+
+### Producer: send messages
+
+```python
+import time
+from cledar.kafka.clients.producer import KafkaProducer
+from cledar.kafka.config.schemas import KafkaProducerConfig
+
+producer = KafkaProducer(
+    KafkaProducerConfig(
+        kafka_servers="localhost:9092",           # or ["host1:9092", "host2:9092"]
+        kafka_group_id="example-producer",
+        kafka_topic_prefix="my-prefix.",          # optional
+        kafka_block_buffer_time_sec=1,
+    )
+)
+
+producer.connect()
+
+producer.send(
+    topic="example-topic",                        # final Kafka topic will include prefix
+    key="msg-1",
+    value='{"id":"1","message":"hello","timestamp": %f}' % time.time(),
+)
+
+# Optionally check connection status
+assert producer.is_alive()
+
+producer.shutdown()
+```
+
+### Consumer: subscribe and consume
+
+```python
+from cledar.kafka.clients.consumer import KafkaConsumer
+from cledar.kafka.config.schemas import KafkaConsumerConfig
+
+consumer = KafkaConsumer(
+    KafkaConsumerConfig(
+        kafka_servers="localhost:9092",
+        kafka_group_id="example-consumer",
+        kafka_offset="earliest",
+        kafka_topic_prefix="my-prefix.",          # optional
+        kafka_block_consumer_time_sec=1,
+    )
+)
+
+consumer.connect()
+consumer.subscribe(["example-topic"])            # subscribes to prefixed topic
+
+msg = consumer.consume_next()                     # returns KafkaMessage | None
+if msg is not None:
+    print(msg.topic, msg.key, msg.value)
+
+assert consumer.is_alive()
+consumer.shutdown()
+```
+
+### Dead Letter Handling
+
+```python
+from cledar.kafka.handlers.dead_letter import DeadLetterHandler
+from cledar.kafka.models.output import FailedMessageData
+
+# Assume you already have a connected producer and a consumed message
+handler = DeadLetterHandler(producer, dlq_topic="errors-topic")
+
+failure_details = [
+    FailedMessageData(
+        raised_at="2024-01-01T00:00:00Z",
+        exception_message="Processing failed",
+        exception_trace="Traceback...",
+        failure_reason="validation_error",
+    )
+]
+
+handler.handle(message, failure_details)
+```
+
+### Parsing to Typed Payloads
+
+```python
+from pydantic import BaseModel
+from cledar.kafka.handlers.parser import InputParser
+
+class Payload(BaseModel):
+    id: str
+    message: str
+
+parser = InputParser(Payload)
+parsed = parser.parse_message(message)  # -> ParsedMessage[Payload]
+print(parsed.payload.id, parsed.payload.message)
+```
+
+## Project Structure
+
+```
+cledar/kafka/
+├── clients/
+│   ├── base.py                 # BaseKafkaClient (shared logic)
+│   ├── consumer.py             # KafkaConsumer wrapper
+│   └── producer.py             # KafkaProducer wrapper
+├── config/
+│   └── schemas.py              # Pydantic frozen dataclass configs
+├── handlers/
+│   ├── dead_letter.py          # DeadLetterHandler
+│   └── parser.py               # InputParser and related utilities
+├── models/
+│   ├── input.py                # Input model definitions
+│   ├── message.py              # KafkaMessage, etc.
+│   └── output.py               # FailedMessageData, etc.
+├── utils/
+│   ├── callbacks.py            # Delivery callbacks
+│   ├── messages.py             # Message utilities (e.g., extract_id_from_value)
+│   └── topics.py               # Topic utilities/helpers
+├── logger.py                   # Module logger
+└── tests/
+    ├── README.md               # Tests documentation (how to run)
+    ├── conftest.py             # Test-wide teardown (thread cleanup)
+    ├── unit/                   # Unit tests (176)
+    └── integration/            # Integration tests (41) with helpers & shared fixtures
+```
+
+## Running Linters
+
+Common commands from repo root:
+
+```bash
+# Format (ruff)
+uv run ruff format .
+
+# Type-check (mypy)
+uv run mypy kafka/
+
+# Optional: pylint
+uv run pylint kafka/
+```
+
+## Running Tests
+
+See `kafka/tests/README.md` for full details. Quick start:
+
+```bash
+# Unit tests
+PYTHONPATH=. uv run pytest kafka/tests/unit/ -v
+
+# Integration tests (requires Docker running)
+PYTHONPATH=. uv run pytest kafka/tests/integration/ -v
+```
+
+- Integration tests use `testcontainers` with Kafka image `confluentinc/cp-kafka:7.4.0`.
+- Shared fixtures live in `kafka/tests/integration/conftest.py`.
+- Helpers (e.g., `consume_until`) live in `kafka/tests/integration/helpers.py`.
+- Test-wide teardown in `kafka/tests/conftest.py` ensures background threads do not block process exit.
+
+## API Overview
+
+### Configs (pydantic dataclasses)
+
+```python
+from cledar.kafka.config.schemas import KafkaProducerConfig, KafkaConsumerConfig
+```
+
+- Validated, frozen configs; construct with required `kafka_servers` and `kafka_group_id`.
+- Optional fields include `kafka_topic_prefix`, timeouts, and intervals.
+
+### Producer
+
+```python
+from cledar.kafka.clients.producer import KafkaProducer
+```
+
+- `connect()` / `shutdown()`
+- `send(topic: str, value: str, key: str | None = None, headers: list[tuple[str, bytes]] | None = None)`
+- `check_connection()` / `is_alive()`
+
+### Consumer
+
+```python
+from cledar.kafka.clients.consumer import KafkaConsumer
+```
+
+- `connect()` / `shutdown()`
+- `subscribe(topics: list[str])`
+- `consume_next() -> KafkaMessage | None`
+- `commit(message: KafkaMessage) -> None`
+- `check_connection()` / `is_alive()`
+
+### Errors
+
+```python
+from kafka.exceptions import (
+    KafkaConnectionError,
+    KafkaProducerNotConnectedError,
+    KafkaConsumerNotConnectedError,
+)
+```
+
+## Notes
+
+- Always run tests with `PYTHONPATH=.` from the repository root to ensure imports resolve.
+- Integration tests require Docker and will pull testcontainers images on first run.
+- Topics are automatically prefixed with `kafka_topic_prefix` if set in configs.
+
+## License
+
+See the main repository LICENSE file.
+
+## Support
+
+For issues, questions, or contributions, please refer to the repository contribution guidelines.

cledar/kafka/__init__.py

@@ -0,0 +1,42 @@
+"""Kafka client and utilities for Cledar SDK."""
+
+from .clients.base import BaseKafkaClient
+from .clients.consumer import KafkaConsumer
+from .clients.producer import KafkaProducer
+from .config.schemas import (
+    KafkaConsumerConfig,
+    KafkaProducerConfig,
+    KafkaSaslMechanism,
+    KafkaSecurityProtocol,
+)
+from .exceptions import (
+    KafkaConnectionError,
+    KafkaConsumerError,
+    KafkaConsumerNotConnectedError,
+    KafkaProducerNotConnectedError,
+)
+from .handlers.dead_letter import DeadLetterHandler
+from .handlers.parser import IncorrectMessageValueError, InputParser
+from .models.input import InputKafkaMessage
+from .models.message import KafkaMessage
+from .models.output import FailedMessageData
+
+__all__ = [
+    "KafkaConsumer",
+    "KafkaProducer",
+    "BaseKafkaClient",
+    "DeadLetterHandler",
+    "InputParser",
+    "IncorrectMessageValueError",
+    "InputKafkaMessage",
+    "FailedMessageData",
+    "KafkaMessage",
+    "KafkaProducerConfig",
+    "KafkaConsumerConfig",
+    "KafkaSecurityProtocol",
+    "KafkaSaslMechanism",
+    "KafkaConnectionError",
+    "KafkaConsumerNotConnectedError",
+    "KafkaProducerNotConnectedError",
+    "KafkaConsumerError",
+]

cledar/kafka/clients/base.py

@@ -0,0 +1,117 @@
+"""Base Kafka client module."""
+
+import threading
+
+from confluent_kafka import Consumer, KafkaException, Producer
+from pydantic import ConfigDict
+from pydantic.dataclasses import dataclass
+
+from ..config.schemas import KafkaConsumerConfig, KafkaProducerConfig
+from ..exceptions import (
+    KafkaConnectionError,
+    KafkaConsumerNotConnectedError,
+    KafkaProducerNotConnectedError,
+)
+from ..logger import logger
+
+
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class BaseKafkaClient:
+    """Base class for Kafka clients.
+
+    This class provides common functionality for both producers and consumers,
+    such as connection monitoring and shutdown handling.
+    """
+
+    config: KafkaProducerConfig | KafkaConsumerConfig
+    client: Producer | Consumer | None = None
+    connection_check_thread: threading.Thread | None = None
+
+    def __post_init__(self) -> None:
+        """Initialize the client with instance-level events."""
+        # Create instance-level stop event
+        self._stop_event = threading.Event()
+        logger.info(
+            f"Initializing {self.__class__.__name__}.", extra={"config": self.config}
+        )
+
+    def start_connection_check_thread(self) -> None:
+        """Start a background thread to monitor the Kafka connection."""
+        if self.connection_check_thread is None:
+            self.connection_check_thread = threading.Thread(
+                target=self._monitor_connection
+            )
+            self.connection_check_thread.start()
+            logger.info(
+                f"Started {self.__class__.__name__} connection check thread.",
+                extra={"interval": self.config.kafka_connection_check_interval_sec},
+            )
+
+    def _monitor_connection(self) -> None:
+        while not self._stop_event.wait(
+            self.config.kafka_connection_check_interval_sec
+        ):
+            try:
+                self.check_connection()
+                logger.info(f"{self.__class__.__name__} connection status: Connected.")
+            except KafkaConnectionError:
+                logger.exception(f"{self.__class__.__name__} connection check failed.")
+
+    def is_alive(self) -> bool:
+        """Check if the client is currently connected and alive.
+
+        Returns:
+            bool: True if connected, False otherwise.
+
+        """
+        try:
+            self.check_connection()
+            return True
+        except (
+            KafkaProducerNotConnectedError,
+            KafkaConsumerNotConnectedError,
+            KafkaConnectionError,
+        ):
+            return False
+
+    def check_connection(self) -> None:
+        """Check the connection to Kafka servers.
+
+        When the broker is not available (or the address is wrong),
+        the 'connection refused' error is not caught.
+        Ref: https://github.com/confluentinc/confluent-kafka-python/issues/941
+        The below is far-from-perfect workaround handling that.
+
+        """
+        if self.client is None:
+            logger.error(
+                f"{self.__class__.__name__} is not connected. Call 'connect' first.",
+            )
+            raise (
+                KafkaProducerNotConnectedError
+                if isinstance(self.config, KafkaProducerConfig)
+                else KafkaConsumerNotConnectedError
+            )
+        try:
+            self.client.list_topics(
+                timeout=self.config.kafka_connection_check_timeout_sec
+            )
+        except KafkaException as exception:
+            logger.exception("Failed to connect to Kafka servers.")
+            raise KafkaConnectionError from exception
+
+    def shutdown(self) -> None:
+        """Shutdown the client and stop connection monitoring."""
+        logger.info("Closing %s...", self.__class__.__name__)
+        self._stop_event.set()
+        if self.connection_check_thread is not None:
+            self.connection_check_thread.join()
+            logger.info("Stopped connection check thread.")
+        if isinstance(self.client, Producer):
+            self.client.flush(-1)
+            logger.info("%s flushed.", self.__class__.__name__)
+        elif isinstance(self.client, Consumer):
+            self.client.close()
+        # Clear the client reference to indicate shutdown
+        self.client = None
+        logger.info("%s closed.", self.__class__.__name__)

cledar/kafka/clients/consumer.py

@@ -0,0 +1,138 @@
+"""Kafka consumer client module."""
+
+from confluent_kafka import Consumer, KafkaException
+from pydantic import ConfigDict
+from pydantic.dataclasses import dataclass
+
+from ..config.schemas import KafkaConsumerConfig
+from ..exceptions import (
+    KafkaConsumerError,
+    KafkaConsumerNotConnectedError,
+)
+from ..logger import logger
+from ..models.message import KafkaMessage
+from ..utils.messages import consumer_not_connected_msg, extract_id_from_value
+from ..utils.topics import build_topic
+from .base import BaseKafkaClient
+
+
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class KafkaConsumer(BaseKafkaClient):
+    """Kafka consumer client.
+
+    This class provides methods to connect to Kafka, subscribe to topics,
+    and consume messages.
+    """
+
+    config: KafkaConsumerConfig
+    client: Consumer | None = None
+
+    def connect(self) -> None:
+        """Connect to Kafka servers and start connection monitoring."""
+        self.client = Consumer(self.config.to_kafka_config())
+        self.check_connection()
+        logger.info(
+            "Connected KafkaConsumer to Kafka servers.",
+            extra={"kafka_servers": self.config.kafka_servers},
+        )
+        self.start_connection_check_thread()
+
+    def subscribe(self, topics: list[str]) -> None:
+        """Subscribe to a list of topics.
+
+        Args:
+            topics: A list of topic names to subscribe to.
+
+        """
+        if self.client is None:
+            logger.error(
+                consumer_not_connected_msg,
+                extra={"topics": topics},
+            )
+            raise KafkaConsumerNotConnectedError
+
+        topics = [
+            build_topic(topic_name=topic, prefix=self.config.kafka_topic_prefix)
+            for topic in topics
+        ]
+
+        try:
+            logger.info(
+                "Subscribing to topics.",
+                extra={"topics": topics},
+            )
+            self.client.subscribe(topics)
+
+        except KafkaException as exception:
+            logger.exception(
+                "Failed to subscribe to topics.",
+                extra={"topics": topics},
+            )
+            raise exception
+
+    def consume_next(self) -> KafkaMessage | None:
+        """Consume the next message from subscribed topics.
+
+        Returns:
+            KafkaMessage | None: The consumed message or None if no message is
+                available.
+
+        """
+        if self.client is None:
+            logger.error(consumer_not_connected_msg)
+            raise KafkaConsumerNotConnectedError
+
+        try:
+            msg = self.client.poll(self.config.kafka_block_consumer_time_sec)
+
+            if msg is None:
+                return None
+
+            if msg.error():
+                logger.error(
+                    "Consumer error.",
+                    extra={"error": msg.error()},
+                )
+                raise KafkaConsumerError(msg.error())
+
+            logger.debug(
+                "Received message.",
+                extra={
+                    "topic": msg.topic(),
+                    "msg_id": extract_id_from_value(msg.value().decode("utf-8")),
+                    "key": msg.key(),
+                },
+            )
+            return KafkaMessage(
+                topic=msg.topic(),
+                value=msg.value().decode("utf-8") if msg.value() else None,
+                key=msg.key().decode("utf-8") if msg.key() else None,
+                offset=msg.offset(),
+                partition=msg.partition(),
+            )
+
+        except KafkaException as exception:
+            logger.exception("Failed to consume message.")
+            raise exception
+
+    def commit(self, message: KafkaMessage) -> None:
+        """Commit offsets for the current message.
+
+        Args:
+            message: The message for which to commit offsets.
+
+        """
+        if self.client is None:
+            logger.error(consumer_not_connected_msg)
+            raise KafkaConsumerNotConnectedError
+
+        try:
+            self.client.commit(asynchronous=True)
+            logger.debug(
+                "Commit requested.",
+                extra={"offset": message.offset, "partition": message.partition},
+            )
+
+        except KafkaException as exception:
+            logger.exception("Failed to commit offsets.")
+            raise exception

cledar/kafka/clients/producer.py

@@ -0,0 +1,97 @@
+"""Kafka producer client module."""
+
+from confluent_kafka import KafkaException, Producer
+from pydantic import ConfigDict
+from pydantic.dataclasses import dataclass
+
+from ..config.schemas import KafkaProducerConfig
+from ..exceptions import KafkaProducerNotConnectedError
+from ..logger import logger
+from ..utils.callbacks import delivery_callback
+from ..utils.messages import extract_id_from_value
+from ..utils.topics import build_topic
+from .base import BaseKafkaClient
+
+
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class KafkaProducer(BaseKafkaClient):
+    """Kafka producer client.
+
+    This class provides methods to connect to Kafka and send messages.
+    """
+
+    config: KafkaProducerConfig
+    client: Producer | None = None
+
+    def connect(self) -> None:
+        """Connect to Kafka servers and start connection monitoring."""
+        self.client = Producer(self.config.to_kafka_config())
+        self.check_connection()
+        logger.info(
+            "Connected Producer to Kafka servers.",
+            extra={"kafka_servers": self.config.kafka_servers},
+        )
+        self.start_connection_check_thread()
+
+    def send(
+        self,
+        topic: str,
+        value: str | None,
+        key: str | None,
+        headers: list[tuple[str, bytes]] | None = None,
+    ) -> None:
+        """Send a message to a Kafka topic.
+
+        Args:
+            topic: The name of the topic to send the message to.
+            value: The message value.
+            key: The message key.
+            headers: Optional list of message headers.
+
+        """
+        if self.client is None:
+            logger.error(
+                "KafkaProducer is not connected. Call 'connect' first.",
+                extra={
+                    "topic": topic,
+                    "msg_id": extract_id_from_value(value),
+                    "key": key,
+                },
+            )
+            raise KafkaProducerNotConnectedError
+
+        topic = build_topic(topic_name=topic, prefix=self.config.kafka_topic_prefix)
+
+        try:
+            logger.debug(
+                "Sending message to topic.",
+                extra={
+                    "topic": topic,
+                    "msg_id": extract_id_from_value(value),
+                    "key": key,
+                    "headers": headers,
+                },
+            )
+            self.client.produce(
+                topic=topic,
+                value=value,
+                key=key,
+                headers=headers,
+                callback=delivery_callback,
+            )
+            self.client.poll(0)
+
+        except BufferError:
+            logger.warning("Buffer full, waiting for free space on the queue")
+            self.client.poll(self.config.kafka_block_buffer_time_sec)
+            self.client.produce(
+                topic=topic,
+                value=value,
+                key=key,
+                headers=headers,
+                callback=delivery_callback,
+            )
+
+        except KafkaException as exception:
+            logger.exception("Failed to send message.")
+            raise exception