cledar-sdk 2.0.2__py3-none-any.whl → 2.0.3__py3-none-any.whl

cledar/kafka/config/schemas.py

@@ -0,0 +1,178 @@
+from enum import Enum
+
+from pydantic import field_validator
+from pydantic.dataclasses import dataclass
+
+
+class KafkaSecurityProtocol(str, Enum):
+    """Supported Kafka security protocols."""
+
+    PLAINTEXT = "PLAINTEXT"
+    SSL = "SSL"
+    SASL_PLAINTEXT = "SASL_PLAINTEXT"
+    SASL_SSL = "SASL_SSL"
+
+
+class KafkaSaslMechanism(str, Enum):
+    """Supported Kafka SASL mechanisms."""
+
+    PLAIN = "PLAIN"
+    SCRAM_SHA_256 = "SCRAM-SHA-256"
+    SCRAM_SHA_512 = "SCRAM-SHA-512"
+    GSSAPI = "GSSAPI"
+
+
+def _validate_kafka_servers(v: list[str] | str) -> list[str] | str:
+    """Validate kafka_servers is not empty."""
+    if isinstance(v, str) and v.strip() == "":
+        raise ValueError("kafka_servers cannot be empty")
+    if isinstance(v, list) and len(v) == 0:
+        raise ValueError("kafka_servers cannot be empty list")
+    return v
+
+
+def _validate_non_negative(v: int) -> int:
+    """Validate that timeout values are non-negative."""
+    if v < 0:
+        raise ValueError("timeout values must be non-negative")
+    return v
+
+
+@dataclass(frozen=True)
+class KafkaProducerConfig:
+    """Configuration for Kafka Producer.
+
+    Args:
+        kafka_servers: List of Kafka broker addresses or comma-separated string
+        kafka_group_id: Consumer group identifier
+        kafka_topic_prefix: Optional prefix for topic names
+        kafka_block_buffer_time_sec: Time to block when buffer is full
+        kafka_connection_check_timeout_sec: Timeout for connection health checks
+        kafka_connection_check_interval_sec: Interval between connection checks
+        kafka_partitioner: Partitioning strategy for messages
+        compression_type: Compression type for messages (gzip, snappy, lz4, zstd,
+            or None)
+    """
+
+    kafka_servers: list[str] | str
+    kafka_group_id: str
+    kafka_security_protocol: KafkaSecurityProtocol | None = None
+    kafka_sasl_mechanism: KafkaSaslMechanism | None = None
+    kafka_sasl_username: str | None = None
+    kafka_sasl_password: str | None = None
+    kafka_topic_prefix: str | None = None
+    kafka_block_buffer_time_sec: int = 10
+    kafka_connection_check_timeout_sec: int = 5
+    kafka_connection_check_interval_sec: int = 60
+    kafka_partitioner: str = "consistent_random"
+    compression_type: str | None = "gzip"
+
+    @field_validator("kafka_servers")
+    @classmethod
+    def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        return _validate_kafka_servers(v)
+
+    @field_validator(
+        "kafka_block_buffer_time_sec",
+        "kafka_connection_check_timeout_sec",
+        "kafka_connection_check_interval_sec",
+    )
+    @classmethod
+    def validate_positive_timeouts(cls, v: int) -> int:
+        return _validate_non_negative(v)
+
+    def to_kafka_config(self) -> dict[str, list[str] | str | None]:
+        """Build Kafka producer configuration dictionary."""
+        config = {
+            "bootstrap.servers": self.kafka_servers,
+            "client.id": self.kafka_group_id,
+            "compression.type": self.compression_type,
+            "partitioner": self.kafka_partitioner,
+        }
+
+        # Add SASL configuration if specified
+        if self.kafka_security_protocol:
+            config["security.protocol"] = self.kafka_security_protocol.value
+            if self.kafka_sasl_mechanism:
+                config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
+            if self.kafka_sasl_username:
+                config["sasl.username"] = self.kafka_sasl_username
+            if self.kafka_sasl_password:
+                config["sasl.password"] = self.kafka_sasl_password
+
+        return config
+
+
+@dataclass(frozen=True)
+class KafkaConsumerConfig:
+    """Configuration for Kafka Consumer.
+
+    Args:
+        kafka_servers: List of Kafka broker addresses or comma-separated string
+        kafka_group_id: Consumer group identifier
+        kafka_offset: Starting offset position ('earliest', 'latest', or specific
+            offset)
+        kafka_topic_prefix: Optional prefix for topic names
+        kafka_block_consumer_time_sec: Time to block waiting for messages
+        kafka_connection_check_timeout_sec: Timeout for connection health checks
+        kafka_auto_commit_interval_ms: Interval for automatic offset commits
+        kafka_connection_check_interval_sec: Interval between connection checks
+    """
+
+    kafka_servers: list[str] | str
+    kafka_group_id: str
+    kafka_security_protocol: KafkaSecurityProtocol | None = None
+    kafka_sasl_mechanism: KafkaSaslMechanism | None = None
+    kafka_sasl_username: str | None = None
+    kafka_sasl_password: str | None = None
+    kafka_offset: str = "latest"
+    kafka_topic_prefix: str | None = None
+    kafka_block_consumer_time_sec: int = 2
+    kafka_connection_check_timeout_sec: int = 5
+    kafka_auto_commit_interval_ms: int = 1000
+    kafka_connection_check_interval_sec: int = 60
+
+    @field_validator("kafka_servers")
+    @classmethod
+    def validate_kafka_servers(cls, v: list[str] | str) -> list[str] | str:
+        return _validate_kafka_servers(v)
+
+    @field_validator("kafka_offset")
+    @classmethod
+    def validate_kafka_offset(cls, v: str) -> str:
+        if v.strip() == "":
+            raise ValueError("kafka_offset cannot be empty")
+        return v
+
+    @field_validator(
+        "kafka_block_consumer_time_sec",
+        "kafka_connection_check_timeout_sec",
+        "kafka_auto_commit_interval_ms",
+        "kafka_connection_check_interval_sec",
+    )
+    @classmethod
+    def validate_positive_timeouts(cls, v: int) -> int:
+        return _validate_non_negative(v)
+
+    def to_kafka_config(self) -> dict[str, int | list[str] | str]:
+        """Build Kafka consumer configuration dictionary."""
+        config = {
+            "bootstrap.servers": self.kafka_servers,
+            "enable.auto.commit": False,
+            "enable.partition.eof": False,
+            "auto.commit.interval.ms": self.kafka_auto_commit_interval_ms,
+            "auto.offset.reset": self.kafka_offset,
+            "group.id": self.kafka_group_id,
+        }
+
+        # Add SASL configuration if specified
+        if self.kafka_security_protocol:
+            config["security.protocol"] = self.kafka_security_protocol.value
+            if self.kafka_sasl_mechanism:
+                config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
+            if self.kafka_sasl_username:
+                config["sasl.username"] = self.kafka_sasl_username
+            if self.kafka_sasl_password:
+                config["sasl.password"] = self.kafka_sasl_password
+
+        return config

cledar/kafka/exceptions.py

@@ -0,0 +1,22 @@
+class KafkaProducerNotConnectedError(Exception):
+    """
+    Custom exception for KafkaProducer to indicate it is not connected.
+    """
+
+
+class KafkaConsumerNotConnectedError(Exception):
+    """
+    Custom exception for KafkaConsumer to indicate it is not connected.
+    """
+
+
+class KafkaConnectionError(Exception):
+    """
+    Custom exception to indicate connection failures.
+    """
+
+
+class KafkaConsumerError(Exception):
+    """
+    Custom exception for KafkaConsumer to indicate errors.
+    """

cledar/kafka/handlers/dead_letter.py

@@ -0,0 +1,82 @@
+import json
+
+from ..clients.producer import KafkaProducer
+from ..logger import logger
+from ..models.message import KafkaMessage
+from ..models.output import FailedMessageData
+
+
+class DeadLetterHandler:
+    """
+    A Handler for handling failed messages and sending them to a DLQ topic.
+    """
+
+    def __init__(self, producer: KafkaProducer, dlq_topic: str) -> None:
+        """
+        Initialize DeadLetterHandler with a Kafka producer and DLQ topic.
+
+        :param producer: KafkaProducer instance.
+        :param dlq_topic: The name of the DLQ Kafka topic.
+        """
+        self.producer: KafkaProducer = producer
+        self.dlq_topic: str = dlq_topic
+
+    def handle(
+        self,
+        message: KafkaMessage,
+        failures_details: list[FailedMessageData] | None,
+    ) -> None:
+        """
+        Handles a failed message by sending it to the DLQ topic.
+
+        :param message: The original Kafka message.
+        :param failures_details: A list of FailedMessageData.
+        """
+        logger.info("Handling message for DLQ.")
+
+        kafka_headers = self._build_headers(failures_details=failures_details)
+
+        logger.info("DLQ message built successfully.")
+        self._send_message(message.value, message.key, kafka_headers)
+
+    def _build_headers(
+        self,
+        failures_details: list[FailedMessageData] | None,
+    ) -> list[tuple[str, bytes]]:
+        """
+        Builds Kafka headers containing exception details.
+
+        :param failures_details: A list of FailedMessageData.
+        :return: A list of Kafka headers.
+        """
+        headers: list[tuple[str, bytes]] = []
+
+        if failures_details:
+            failures_json = json.dumps(
+                [failure.model_dump() for failure in failures_details]
+            )
+            headers.append(("failures_details", failures_json.encode("utf-8")))
+
+        return headers
+
+    def _send_message(
+        self,
+        message: str | None,
+        key: str | None,
+        headers: list[tuple[str, bytes]],
+    ) -> None:
+        """
+        Sends a DLQ message to the Kafka DLQ topic with headers.
+
+        :param message: The DLQ message payload.
+        :param key: The original Kafka message key.
+        :param headers: Kafka headers containing exception details.
+        """
+        self.producer.send(
+            topic=self.dlq_topic, value=message, key=key, headers=headers
+        )
+        logger.info(
+            "Message sent to DLQ topic successfully with key: %s and headers: %s",
+            key,
+            headers,
+        )

cledar/kafka/handlers/parser.py

@@ -0,0 +1,49 @@
+import json
+
+from pydantic import BaseModel
+
+from ..models.input import (
+    InputKafkaMessage,
+)
+from ..models.message import KafkaMessage
+
+
+class IncorrectMessageValueError(Exception):
+    """
+    Message needs to have `value` field present in order to be parsed.
+
+    This is unless `model` is set to `None`.
+    """
+
+
+class InputParser[Payload: BaseModel]:
+    def __init__(self, model: type[Payload]) -> None:
+        self.model: type[Payload] = model
+
+    def parse_json(self, json_str: str) -> Payload:
+        """Parse JSON text and validate into the target Payload model.
+
+        Invalid JSON should raise IncorrectMessageValueError, while schema
+        validation errors should bubble up as ValidationError.
+        """
+        try:
+            data = json.loads(json_str)
+        except json.JSONDecodeError as exc:
+            # Normalize invalid JSON into our domain-specific error
+            raise IncorrectMessageValueError from exc
+        return self.model.model_validate(data)
+
+    def parse_message(self, message: KafkaMessage) -> InputKafkaMessage[Payload]:
+        if message.value is None and self.model is not None:
+            raise IncorrectMessageValueError
+
+        obj = self.parse_json(message.value)
+
+        return InputKafkaMessage(
+            key=message.key,
+            value=message.value,
+            payload=obj,
+            topic=message.topic,
+            offset=message.offset,
+            partition=message.partition,
+        )

cledar/kafka/logger.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("kafka_service")

cledar/kafka/models/input.py

@@ -0,0 +1,13 @@
+import dataclasses
+from typing import TypeVar
+
+from pydantic import BaseModel
+
+from .message import KafkaMessage
+
+Payload = TypeVar("Payload", bound=BaseModel)
+
+
+@dataclasses.dataclass
+class InputKafkaMessage[Payload](KafkaMessage):
+    payload: Payload

cledar/kafka/models/message.py

@@ -0,0 +1,10 @@
+from pydantic.dataclasses import dataclass
+
+
+@dataclass
+class KafkaMessage:
+    topic: str
+    value: str | None
+    key: str | None
+    offset: int | None
+    partition: int | None

cledar/kafka/models/output.py

@@ -0,0 +1,8 @@
+import pydantic
+
+
+class FailedMessageData(pydantic.BaseModel):
+    raised_at: str
+    exception_message: str | None
+    exception_trace: str | None
+    failure_reason: str | None

cledar/kafka/tests/.env.test.kafka

@@ -0,0 +1,3 @@
+KAFKA_SERVERS="localhost:9092"
+KAFKA_GROUP_ID="test-group"
+KAFKA_TOPIC_PREFIX="test."

cledar/kafka/tests/README.md

@@ -0,0 +1,216 @@
+# Kafka Service Tests
+
+This directory contains the test suite for the Kafka service, organized into unit and integration tests.
+
+## Directory Structure
+
+```
+tests/
+├── conftest.py              # Test-wide teardown (cleans Kafka client threads)
+├── README.md
+├── unit/                    # Unit tests (176 tests)
+│   ├── test_base_kafka_client.py
+│   ├── test_config_validation.py
+│   ├── test_dead_letter_handler.py
+│   ├── test_error_handling.py
+│   ├── test_input_parser.py
+│   ├── test_input_parser_comprehensive.py
+│   ├── test_utils.py
+│   ├── test_utils_comprehensive.py
+│   └── requirements-test.txt
+└── integration/             # Integration tests (41 tests)
+    ├── conftest.py          # Shared Kafka fixtures (container, configs, clients)
+    ├── helpers.py           # E2EData, consume_until, ensure_topic_and_subscribe
+    ├── test_integration.py
+    ├── test_producer_integration.py
+    ├── test_consumer_integration.py
+    └── test_producer_consumer_interaction.py
+```
+
+## Test Categories
+
+### Unit Tests (`unit/`)
+Unit tests focus on testing individual components in isolation using mocks and stubs. They are fast, reliable, and don't require external dependencies.
+
+- **Base Client Tests**: Test the base Kafka client functionality
+- **Config Validation**: Test configuration validation and schema validation
+- **Dead Letter Handler**: Test dead letter queue handling with mocked producers
+- **Error Handling**: Test error scenarios and exception handling
+- **Input Parser**: Test message parsing and validation
+- **Utils**: Test utility functions and helper methods
+
+### Integration Tests (`integration/`)
+Integration tests use real external dependencies (like Kafka via testcontainers) to test the complete flow of the system.
+
+- **Real Kafka Integration**: Tests with actual Kafka instance using testcontainers
+- **Producer Integration**: Real producer operations and message sending
+- **Consumer Integration**: Real consumer operations and message consumption
+- **Producer-Consumer Interaction**: Real interaction patterns between producer and consumer
+- **End-to-End Flows**: Complete producer-consumer workflows
+- **Connection Recovery**: Real connection failure and recovery scenarios
+- **Performance Tests**: Stress tests and large message handling
+
+## Running Tests
+
+### Run All Tests
+```bash
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/
+```
+
+### Run Only Unit Tests
+```bash
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/unit/
+```
+
+### Run Only Integration Tests
+```bash
+# Run all integration tests
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/
+
+# Run specific integration test file
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_producer_integration.py
+
+# Run single test
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_integration.py::test_end_to_end_message_flow -v
+```
+
+### Run Specific Test Files
+```bash
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/unit/test_config_validation.py
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_integration.py
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_producer_integration.py
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_consumer_integration.py
+```
+
+## Test Requirements
+
+- **Unit Tests**: No external dependencies required
+- **Integration Tests**: Requires Docker to be running for testcontainers
+- **Slow Integration Tests**: Marked as skipped by default due to execution time (2-5 minutes each)
+
+## Performance Notes
+
+- **Unit Tests**: Fast execution (~10–15 seconds for all 176 tests)
+- **Integration Tests**: Moderate execution (~2–2.5 minutes for 41 tests)
+- Helpers reduce flakiness: `consume_until()` polls with timeout instead of fixed sleeps
+
+## Docker Setup for Integration Tests
+
+The integration tests use testcontainers to spin up real Kafka instances for testing. This requires Docker to be installed and running.
+
+### Prerequisites
+
+1. **Install Docker Desktop** (recommended):
+   - [Docker Desktop for Mac](https://docs.docker.com/desktop/mac/install/)
+   - [Docker Desktop for Windows](https://docs.docker.com/desktop/windows/install/)
+   - [Docker Desktop for Linux](https://docs.docker.com/desktop/linux/install/)
+
+2. **Or install Docker Engine** (alternative):
+   ```bash
+   # macOS (using Homebrew)
+   brew install docker
+   
+   # Ubuntu/Debian
+   sudo apt-get update
+   sudo apt-get install docker.io
+   
+   # CentOS/RHEL
+   sudo yum install docker
+   ```
+
+### Starting Docker
+
+#### Docker Desktop
+1. Launch Docker Desktop application
+2. Wait for Docker to start (you'll see the Docker whale icon in your system tray)
+3. Verify Docker is running:
+   ```bash
+   docker --version
+   docker ps
+   ```
+
+#### Docker Engine (Linux)
+```bash
+# Start Docker service
+sudo systemctl start docker
+sudo systemctl enable docker
+
+# Add your user to docker group (optional, to avoid sudo)
+sudo usermod -aG docker $USER
+# Log out and back in for group changes to take effect
+
+# Verify Docker is running
+docker --version
+docker ps
+```
+
+### Running Integration Tests
+
+Once Docker is running, you can execute the integration tests:
+
+```bash
+# Run all integration tests
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/
+
+# Run a specific integration test
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/test_integration.py::test_producer_consumer_basic_flow
+
+# Run integration tests with verbose output
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/ -v
+
+# Run integration tests and show logs
+PYTHONPATH=. uv run pytest cledar/kafka_service/tests/integration/ -s
+```
+
+### Troubleshooting Docker Issues
+
+#### Docker not running
+```bash
+# Check if Docker is running
+docker ps
+# If you get "Cannot connect to the Docker daemon", Docker is not running
+
+# Start Docker Desktop or Docker service
+# Docker Desktop: Launch the application
+# Docker Engine: sudo systemctl start docker
+```
+
+#### Permission denied errors
+```bash
+# Add user to docker group (Linux)
+sudo usermod -aG docker $USER
+# Log out and back in
+
+# Or run with sudo (not recommended)
+sudo docker ps
+```
+
+#### Port conflicts
+If you have Kafka running locally on port 9092, the testcontainers will automatically use different ports. No action needed.
+
+#### Resource constraints
+If tests fail due to memory/CPU constraints:
+```bash
+# Check Docker resource limits in Docker Desktop settings
+# Increase memory allocation if needed (recommended: 4GB+)
+```
+
+### Test Container Details
+
+The integration tests use:
+- **Kafka Image**: `confluentinc/cp-kafka:7.4.0`
+- **Automatic Port Assignment**: testcontainers handles port conflicts
+- **Automatic Cleanup**: containers are removed after tests complete
+- **Session Scope**: Kafka container is shared across all integration tests in a session
+
+## Test Statistics
+
+- **Total Tests**: 217
+- **Unit Tests**: 176
+- **Integration Tests**: 41
+
+## Notes
+
+- All tests use `PYTHONPATH=.` to ensure proper module imports
+- Integration tests use shared fixtures in `integration/conftest.py` and helpers in `integration/helpers.py`
+- Test-wide teardown in `tests/conftest.py` ensures Kafka client threads don’t block process exit

cledar/kafka/tests/conftest.py

@@ -0,0 +1,104 @@
+"""
+Pytest configuration and fixtures to ensure proper teardown of Kafka clients and
+their background threads during tests.
+"""
+
+import threading
+import time
+import weakref
+from collections.abc import Callable, Generator
+from typing import Any, cast
+
+import pytest
+
+from cledar.kafka.clients.base import BaseKafkaClient
+from cledar.kafka.logger import logger
+
+# Weak registry of all created BaseKafkaClient instances (does not keep them alive)
+_active_clients: "weakref.WeakSet[BaseKafkaClient]" = weakref.WeakSet()
+
+
+def _wrap_post_init() -> tuple[
+    Callable[[BaseKafkaClient], None], Callable[[BaseKafkaClient], None]
+]:
+    """Monkeypatch BaseKafkaClient.__post_init__ to register instances."""
+    original = BaseKafkaClient.__post_init__
+
+    def wrapped(self: BaseKafkaClient) -> None:
+        original(self)
+        try:
+            _active_clients.add(self)
+        except Exception:
+            # Best-effort registration only for tests
+            pass
+
+    return original, wrapped
+
+
+def _wrap_start_connection_check_thread() -> tuple[
+    Callable[[BaseKafkaClient], None], Callable[[BaseKafkaClient], None]
+]:
+    """Monkeypatch start_connection_check_thread to use daemon threads in tests.
+
+    This prevents non-daemon threads from blocking interpreter shutdown when tests
+    forget to call shutdown() explicitly.
+    """
+    original = BaseKafkaClient.start_connection_check_thread
+
+    def wrapped(self: BaseKafkaClient) -> None:
+        if self.connection_check_thread is None:
+            self.connection_check_thread = threading.Thread(
+                target=self._monitor_connection
+            )
+            # Ensure test background threads never block process exit
+            self.connection_check_thread.daemon = True
+            self.connection_check_thread.start()
+            logger.info(
+                f"Started {self.__class__.__name__} connection check thread.",
+                extra={"interval": self.config.kafka_connection_check_interval_sec},
+            )
+
+    return original, wrapped
+
+
+def _cleanup_all_clients() -> None:
+    """Shutdown all known clients; ignore errors during cleanup."""
+    for client in list(_active_clients):
+        try:
+            client.shutdown()
+        except Exception:
+            pass
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _session_monkeypatch() -> Generator[None, None, None]:
+    """Apply monkeypatches for the entire test session and ensure final cleanup."""
+    # Monkeypatch __post_init__ to register instances
+    orig_post_init, wrapped_post_init = _wrap_post_init()
+    cast(Any, BaseKafkaClient).__post_init__ = wrapped_post_init
+
+    # Monkeypatch start_connection_check_thread to create daemon threads
+    orig_start, wrapped_start = _wrap_start_connection_check_thread()
+    cast(Any, BaseKafkaClient).start_connection_check_thread = wrapped_start
+
+    try:
+        yield
+    finally:
+        # Restore originals
+        cast(Any, BaseKafkaClient).__post_init__ = orig_post_init
+        cast(Any, BaseKafkaClient).start_connection_check_thread = orig_start
+
+        # Final cleanup at session end
+        _cleanup_all_clients()
+
+        # Give threads a small grace period to finish
+        time.sleep(0.1)
+
+
+@pytest.fixture(autouse=True)
+def _per_test_cleanup() -> Generator[None, None, None]:
+    """Ensure all clients are shut down after each individual test."""
+    yield
+    _cleanup_all_clients()
+    # Small grace to allow quick thread exit
+    time.sleep(0.05)