cledar-sdk 2.0.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

cledar/kafka/tests/integration/test_producer_consumer_interaction.py

@@ -0,0 +1,388 @@
+"""
+Integration tests for producer-consumer interaction patterns,
+message ordering, error handling, and complex scenarios using real Kafka.
+"""
+
+import json
+import time
+from collections.abc import Generator
+
+import pytest
+
+from cledar.kafka.clients.consumer import KafkaConsumer
+from cledar.kafka.clients.producer import KafkaProducer
+from cledar.kafka.config.schemas import KafkaConsumerConfig, KafkaProducerConfig
+from cledar.kafka.handlers.dead_letter import DeadLetterHandler
+from cledar.kafka.handlers.parser import InputParser
+from cledar.kafka.models.output import FailedMessageData
+from cledar.kafka.tests.integration.helpers import E2EData, consume_until
+
+
+class InteractionTestData(E2EData):
+    def to_json(self) -> str:  # keep existing API usage
+        return super().to_json()
+
+
+@pytest.fixture
+def producer_config(kafka_bootstrap_servers: str) -> KafkaProducerConfig:
+    """Create producer configuration for integration tests."""
+    return KafkaProducerConfig(
+        kafka_servers=kafka_bootstrap_servers,
+        kafka_group_id="integration-test-producer",
+        kafka_topic_prefix="integration-test.",
+        kafka_block_buffer_time_sec=1,
+        kafka_connection_check_timeout_sec=5,
+        kafka_connection_check_interval_sec=10,
+    )
+
+
+@pytest.fixture
+def consumer_config(kafka_bootstrap_servers: str) -> KafkaConsumerConfig:
+    """Create consumer configuration for integration tests."""
+    return KafkaConsumerConfig(
+        kafka_servers=kafka_bootstrap_servers,
+        kafka_group_id="integration-test-consumer",
+        kafka_offset="earliest",
+        kafka_topic_prefix="integration-test.",
+        kafka_block_consumer_time_sec=1,
+        kafka_connection_check_timeout_sec=5,
+        kafka_auto_commit_interval_ms=1000,
+        kafka_connection_check_interval_sec=10,
+    )
+
+
+@pytest.fixture
+def producer(
+    producer_config: KafkaProducerConfig,
+) -> Generator[KafkaProducer, None, None]:
+    """Create and connect a Kafka producer."""
+    producer = KafkaProducer(producer_config)
+    producer.connect()
+    yield producer
+    producer.shutdown()
+
+
+@pytest.fixture
+def consumer(
+    consumer_config: KafkaConsumerConfig,
+) -> Generator[KafkaConsumer, None, None]:
+    """Create and connect a Kafka consumer."""
+    consumer = KafkaConsumer(consumer_config)
+    consumer.connect()
+    yield consumer
+    consumer.shutdown()
+
+
+def test_producer_consumer_basic_interaction(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test basic producer-consumer interaction with real Kafka."""
+    topic = "test-basic-interaction"
+
+    # Send a message first to create the topic
+    test_data = InteractionTestData(
+        id="test-1", message="Hello Kafka!", timestamp=time.time()
+    )
+    producer.send(topic=topic, value=test_data.to_json(), key="test-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume the message
+    message = consumer.consume_next()
+    assert message is not None
+    assert message.topic == f"integration-test.{topic}"
+    assert message.key == "test-key"
+    assert message.value == test_data.to_json()
+
+
+def test_producer_consumer_multiple_messages(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test producer-consumer interaction with multiple messages."""
+    topic = "test-multiple-messages"
+
+    # Send multiple messages first to create the topic
+    messages = []
+    for i in range(5):
+        test_data = InteractionTestData(
+            id=f"test-{i}", message=f"Message {i}", timestamp=time.time()
+        )
+        messages.append(test_data)
+        producer.send(topic=topic, value=test_data.to_json(), key=f"key-{i}")
+
+    # Wait for topic to be created and messages to be sent
+    time.sleep(3)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume all messages
+    received_messages = consume_until(consumer, expected_count=5, timeout_seconds=8)
+
+    assert len(received_messages) == 5
+
+    # Verify message content
+    for i, message in enumerate(received_messages):
+        assert message.topic == f"integration-test.{topic}"
+        assert message.key == f"key-{i}"
+        assert message.value == messages[i].to_json()
+
+
+def test_producer_consumer_with_parser(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test producer-consumer interaction with message parsing."""
+    topic = "test-parser-interaction"
+
+    # Create parser
+    parser = InputParser(InteractionTestData)
+
+    # Send a message first to create the topic
+    test_data = InteractionTestData(
+        id="test-parse", message="Parsed message", timestamp=time.time()
+    )
+    producer.send(topic=topic, value=test_data.to_json(), key="parse-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume and parse the message
+    message = consumer.consume_next()
+    assert message is not None
+
+    parsed_message = parser.parse_message(message)
+    assert parsed_message.payload.id == "test-parse"
+    assert parsed_message.payload.message == "Parsed message"
+
+
+def test_producer_consumer_error_handling(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test error handling in producer-consumer interaction."""
+    topic = "test-error-handling"
+
+    # Send a message with invalid JSON first to create the topic
+    invalid_json = (
+        '{"id": "test", "message": "invalid json", "timestamp": "not-a-number"}'
+    )
+    producer.send(topic=topic, value=invalid_json, key="error-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume the message
+    message = consumer.consume_next()
+    assert message is not None
+    assert message.value == invalid_json
+
+
+def test_producer_consumer_with_dead_letter_handler(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test producer-consumer interaction with dead letter handler."""
+    topic = "test-dlq-interaction"
+    dlq_topic = "test-dlq-topic"
+
+    # Create dead letter handler
+    dlq_handler = DeadLetterHandler(producer, dlq_topic)
+
+    # Send a message first to create the topic
+    test_data = InteractionTestData(
+        id="test-dlq", message="DLQ test message", timestamp=time.time()
+    )
+    producer.send(topic=topic, value=test_data.to_json(), key="dlq-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume the message
+    message = consumer.consume_next()
+    assert message is not None
+
+    # Simulate processing failure and send to DLQ
+    failure_details = [
+        FailedMessageData(
+            raised_at="2024-01-01T00:00:00Z",
+            exception_message="Test processing error",
+            exception_trace="Traceback...",
+            failure_reason="Test failure",
+        )
+    ]
+
+    dlq_handler.handle(message, failure_details)
+
+    # Wait for DLQ message to be sent
+    time.sleep(1)
+
+    # Subscribe to DLQ topic and verify message was sent there
+    consumer.subscribe([dlq_topic])
+    dlq_message = consumer.consume_next()
+
+    assert dlq_message is not None
+    assert dlq_message.topic == f"integration-test.{dlq_topic}"
+    assert dlq_message.value == test_data.to_json()
+
+
+def test_producer_consumer_message_ordering(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test that messages are consumed in the order they were sent."""
+    topic = "test-message-ordering"
+
+    # Send messages with sequential IDs first to create the topic
+    messages = []
+    for i in range(10):
+        test_data = InteractionTestData(
+            id=f"order-{i:03d}", message=f"Ordered message {i}", timestamp=time.time()
+        )
+        messages.append(test_data)
+        producer.send(topic=topic, value=test_data.to_json(), key=f"order-key-{i}")
+
+    # Wait for topic to be created and messages to be sent
+    time.sleep(3)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume messages and verify order
+    received_messages = consume_until(consumer, expected_count=10, timeout_seconds=10)
+
+    assert len(received_messages) == 10
+
+    # Verify messages are in order (by checking the ID in the JSON)
+    for i, message in enumerate(received_messages):
+        parsed_data = json.loads(message.value or "{}")
+        assert parsed_data["id"] == f"order-{i:03d}"
+
+
+def test_producer_consumer_with_headers(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test producer-consumer interaction with message headers."""
+    topic = "test-headers-interaction"
+
+    # Send a message with headers first to create the topic
+    test_data = InteractionTestData(
+        id="test-headers", message="Message with headers", timestamp=time.time()
+    )
+    headers = [
+        ("custom-header", b"custom-value"),
+        ("another-header", b"another-value"),
+    ]
+
+    producer.send(
+        topic=topic,
+        value=test_data.to_json(),
+        key="headers-key",
+        headers=headers,
+    )
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume the message
+    message = consumer.consume_next()
+    assert message is not None
+    assert message.value == test_data.to_json()
+
+    # Validate received value; header visibility varies by consumer implementation.
+
+
+def test_producer_consumer_commit_behavior(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test consumer commit behavior."""
+    topic = "test-commit-behavior"
+
+    # Send a message first to create the topic
+    test_data = InteractionTestData(
+        id="test-commit", message="Commit test message", timestamp=time.time()
+    )
+    producer.send(topic=topic, value=test_data.to_json(), key="commit-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Consume the message
+    message = consumer.consume_next()
+    assert message is not None
+
+    # Commit the message
+    consumer.commit(message)
+
+    # Verify commit was successful (no exception raised)
+    assert True  # If we get here, commit succeeded
+
+
+def test_producer_consumer_connection_recovery(
+    producer: KafkaProducer, consumer: KafkaConsumer
+) -> None:
+    """Test connection recovery behavior."""
+    topic = "test-connection-recovery"
+
+    # Send a message first to create the topic
+    test_data = InteractionTestData(
+        id="test-recovery", message="Recovery test message", timestamp=time.time()
+    )
+    producer.send(topic=topic, value=test_data.to_json(), key="recovery-key")
+
+    # Wait for topic to be created and message to be sent
+    time.sleep(2)
+
+    # Subscribe consumer to topic
+    consumer.subscribe([topic])
+
+    # Wait for subscription to take effect
+    time.sleep(1)
+
+    # Verify both producer and consumer are still alive
+    assert producer.is_alive()
+    assert consumer.is_alive()
+
+    # Consume the message to verify everything is working
+    message = consumer.consume_next()
+    assert message is not None
+    assert message.value == test_data.to_json()

cledar/kafka/tests/integration/test_producer_integration.py

@@ -0,0 +1,217 @@
+"""
+Integration tests for Kafka producer using real Kafka instance.
+"""
+
+import time
+from collections.abc import Generator
+
+import pytest
+
+from cledar.kafka.clients.producer import KafkaProducer
+from cledar.kafka.config.schemas import KafkaProducerConfig
+from cledar.kafka.exceptions import KafkaProducerNotConnectedError
+
+
+@pytest.fixture
+def producer_config(kafka_bootstrap_servers: str) -> KafkaProducerConfig:
+    """Create producer configuration for integration tests."""
+    return KafkaProducerConfig(
+        kafka_servers=kafka_bootstrap_servers,
+        kafka_group_id="integration-test-producer",
+        kafka_topic_prefix="integration-test.",
+        kafka_block_buffer_time_sec=1,
+        kafka_connection_check_timeout_sec=5,
+        kafka_connection_check_interval_sec=10,
+    )
+
+
+@pytest.fixture
+def producer(
+    producer_config: KafkaProducerConfig,
+) -> Generator[KafkaProducer, None, None]:
+    """Create and connect a Kafka producer."""
+    producer = KafkaProducer(producer_config)
+    producer.connect()
+    yield producer
+    producer.shutdown()
+
+
+def test_producer_connect_and_send(producer: KafkaProducer) -> None:
+    """Test producer connection and basic message sending."""
+    topic = "test-producer-basic"
+    test_value = '{"id": "test-1", "message": "Hello Kafka!"}'
+    test_key = "test-key"
+
+    # Send message
+    producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for message to be sent
+    time.sleep(1)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_send_multiple_messages(producer: KafkaProducer) -> None:
+    """Test sending multiple messages."""
+    topic = "test-producer-multiple"
+
+    # Send multiple messages
+    for i in range(5):
+        test_value = f'{{"id": "test-{i}", "message": "Message {i}"}}'
+        test_key = f"key-{i}"
+        producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for messages to be sent
+    time.sleep(2)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_send_with_headers(producer: KafkaProducer) -> None:
+    """Test sending messages with headers."""
+    topic = "test-producer-headers"
+    test_value = '{"id": "test-headers", "message": "Message with headers"}'
+    test_key = "headers-key"
+    headers = [
+        ("custom-header", b"custom-value"),
+        ("another-header", b"another-value"),
+    ]
+
+    # Send message with headers
+    producer.send(topic=topic, value=test_value, key=test_key, headers=headers)
+
+    # Wait for message to be sent
+    time.sleep(1)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_send_large_message(producer: KafkaProducer) -> None:
+    """Test sending large messages."""
+    topic = "test-producer-large"
+
+    # Create a large message
+    large_data = "x" * 10000
+    test_value = f'{{"id": "test-large", "data": "{large_data}"}}'
+    test_key = "large-key"
+
+    # Send large message
+    producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for message to be sent
+    time.sleep(2)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_send_with_special_characters(producer: KafkaProducer) -> None:
+    """Test sending messages with special characters."""
+    topic = "test-producer-special-chars"
+    test_value = '{"id": "test-special", "message": "Special chars: @#$%^&*()"}'
+    test_key = "special-key-with-chars: @#$%^&*()"
+
+    # Send message with special characters
+    producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for message to be sent
+    time.sleep(1)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_send_with_unicode(producer: KafkaProducer) -> None:
+    """Test sending messages with unicode characters."""
+    topic = "test-producer-unicode"
+    test_value = '{"id": "test-unicode", "message": "Unicode: 测试名称"}'
+    test_key = "unicode-key-测试"
+
+    # Send message with unicode
+    producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for message to be sent
+    time.sleep(1)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_connection_check(producer: KafkaProducer) -> None:
+    """Test producer connection checking."""
+    # Verify producer is connected
+    assert producer.is_alive()
+
+    # Check connection explicitly
+    producer.check_connection()
+
+    # Should not raise any exception
+    assert True
+
+
+def test_producer_not_connected_error() -> None:
+    """Test producer error when not connected."""
+    config = KafkaProducerConfig(
+        kafka_servers="localhost:9092",
+        kafka_group_id="test-group",
+    )
+
+    producer = KafkaProducer(config)
+
+    # Should raise error when trying to send without connecting
+    with pytest.raises(KafkaProducerNotConnectedError):
+        producer.send(topic="test", value="test", key="key")
+
+
+def test_producer_shutdown(producer_config: KafkaProducerConfig) -> None:
+    """Test producer shutdown."""
+    producer = KafkaProducer(producer_config)
+    producer.connect()
+
+    # Verify producer is connected
+    assert producer.is_alive()
+
+    # Shutdown producer
+    producer.shutdown()
+
+    # Wait a moment for shutdown to complete
+    time.sleep(0.5)
+
+    # Producer should be disconnected after shutdown
+    assert not producer.is_alive()
+
+
+def test_producer_buffer_handling(producer: KafkaProducer) -> None:
+    """Test producer buffer handling with rapid message sending."""
+    topic = "test-producer-buffer"
+
+    # Send many messages rapidly
+    for i in range(100):
+        test_value = f'{{"id": "buffer-test-{i}", "message": "Buffer test {i}"}}'
+        test_key = f"buffer-key-{i}"
+        producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for messages to be processed
+    time.sleep(3)
+
+    # Verify producer is still connected
+    assert producer.is_alive()
+
+
+def test_producer_topic_prefix(producer: KafkaProducer) -> None:
+    """Test producer topic prefix functionality."""
+    topic = "test-prefix"
+    test_value = '{"id": "test-prefix", "message": "Prefix test"}'
+    test_key = "prefix-key"
+
+    # Send message (should use prefix from config)
+    producer.send(topic=topic, value=test_value, key=test_key)
+
+    # Wait for message to be sent
+    time.sleep(1)
+
+    # Verify producer is still connected
+    assert producer.is_alive()

cledar/kafka/tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for kafka_service."""