sify-queue-kafka 0.1.3__tar.gz → 0.1.4__tar.gz

{sify_queue_kafka-0.1.3 → sify_queue_kafka-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sify-queue-kafka
-Version: 0.1.3
+Version: 0.1.4
 Summary: Enterprise Queue management Framework
 Author: sifymodernization
 Author-email: sifymodernization <sifymodernization.dev@sifycorp.com>
@@ -192,32 +192,33 @@ if __name__ == "__main__":
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `bootstrap_servers` | str | Required | Kafka bootstrap servers |
-| `group_id` | str | None | Consumer group ID |
+| `bootstrap_servers` | str | "localhost:9092" | Kafka bootstrap servers |
+| `group_id` | str | "" | Consumer group ID. Is generated with the tenant_id and source from the consumer config |
 | `enable_auto_commit` | bool | False | Enable auto commit of offsets |
 | `auto_offset_reset` | str | "earliest" | Offset reset policy |
 | `enable_idempotency` | bool | True | Enable duplicate detection |
 | `enable_retry` | bool | True | Enable retry with backoff |
 | `dead_letter_topic` | str | None | Topic for failed events |
-| `stage` | str | None | Processing stage name |
+| `stage` | str | "" | Processing stage name |
+| `enable_idempotent_producer` | bool | False | Enable idempotent producer |
 
 ## Event Model
 
 ```python
 class KafkaEvent(BaseModel):
-    eventId: str 
+    eventId: str = Field(default_factory=lambda: str(uuid.uuid4()))
     eventType: str
     eventVersion: str
     source: str
     tenantId: str
-    timestamp: datetime
-    correlationId: Optional[str]
-    traceId: Optional[str]
-    priority: Optional[str]
-    retryCount: int
-    maxRetries: int
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    correlationId: Optional[str] = None
+    traceId: Optional[str] = None
+    priority: Optional[str] = "MEDIUM"
+    retryCount: int = 0
+    maxRetries: int = 5
     payload: Dict[str, Any]
-    metadata: Optional[Dict[str, Any]]
+    metadata: Optional[Dict[str, Any]] = None
 ```
 
 ## Exception Types
@@ -260,7 +261,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
-### v0.1.3
+### v0.1.4
 - Initial release
 - Basic producer/consumer functionality
 - Event validation with Pydantic

{sify_queue_kafka-0.1.3 → sify_queue_kafka-0.1.4}/README.md

@@ -159,32 +159,33 @@ if __name__ == "__main__":
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `bootstrap_servers` | str | Required | Kafka bootstrap servers |
-| `group_id` | str | None | Consumer group ID |
+| `bootstrap_servers` | str | "localhost:9092" | Kafka bootstrap servers |
+| `group_id` | str | "" | Consumer group ID. Is generated with the tenant_id and source from the consumer config |
 | `enable_auto_commit` | bool | False | Enable auto commit of offsets |
 | `auto_offset_reset` | str | "earliest" | Offset reset policy |
 | `enable_idempotency` | bool | True | Enable duplicate detection |
 | `enable_retry` | bool | True | Enable retry with backoff |
 | `dead_letter_topic` | str | None | Topic for failed events |
-| `stage` | str | None | Processing stage name |
+| `stage` | str | "" | Processing stage name |
+| `enable_idempotent_producer` | bool | False | Enable idempotent producer |
 
 ## Event Model
 
 ```python
 class KafkaEvent(BaseModel):
-    eventId: str 
+    eventId: str = Field(default_factory=lambda: str(uuid.uuid4()))
     eventType: str
     eventVersion: str
     source: str
     tenantId: str
-    timestamp: datetime
-    correlationId: Optional[str]
-    traceId: Optional[str]
-    priority: Optional[str]
-    retryCount: int
-    maxRetries: int
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    correlationId: Optional[str] = None
+    traceId: Optional[str] = None
+    priority: Optional[str] = "MEDIUM"
+    retryCount: int = 0
+    maxRetries: int = 5
     payload: Dict[str, Any]
-    metadata: Optional[Dict[str, Any]]
+    metadata: Optional[Dict[str, Any]] = None
 ```
 
 ## Exception Types
@@ -227,7 +228,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
-### v0.1.3
+### v0.1.4
 - Initial release
 - Basic producer/consumer functionality
 - Event validation with Pydantic

{sify_queue_kafka-0.1.3 → sify_queue_kafka-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sify-queue-kafka"
-version = "0.1.3"
+version = "0.1.4"
 description = "Enterprise Queue management Framework"
 readme = "README.md"
 requires-python = ">=3.8"

sify_queue_kafka-0.1.4/queue_kafka/__init__.py

@@ -0,0 +1,40 @@
+"""
+Sify Queue Kafka
+"""
+
+__version__ = "0.1.4"
+
+from .config import QueueConfig
+from .models import KafkaEvent
+from .producer import Producer
+from .consumer import Consumer
+from .dispatcher import Dispatcher
+from .decorators import event_handler
+from .registry import EventRegistry, registry
+from .exceptions import (
+    QueueSDKError,
+    ConfigurationError,
+    HandlerNotFoundError,
+    EventProcessingError,
+    ProducerError,
+    ConsumerError,
+    SerializationError,
+)
+
+__all__ = [
+    "QueueConfig",
+    "KafkaEvent",
+    "Producer",
+    "Consumer",
+    "Dispatcher",
+    "event_handler",
+    "registry",
+    "EventRegistry",
+    "QueueSDKError",
+    "ConfigurationError",
+    "HandlerNotFoundError",
+    "EventProcessingError",
+    "ProducerError",
+    "ConsumerError",
+    "SerializationError",
+]

sify_queue_kafka-0.1.4/queue_kafka/config.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+@dataclass
+class QueueConfig:
+    bootstrap_servers: str = "localhost:9092"
+    group_id: str = ""
+    enable_auto_commit: bool = False
+    auto_offset_reset: str = "earliest"
+
+    enable_idempotency: bool = True
+    enable_retry: bool = True
+    dead_letter_topic: str = None
+    stage: str = ""
+    
+    enable_idempotent_producer: bool = False  

sify_queue_kafka-0.1.4/queue_kafka/consumer.py

@@ -0,0 +1,101 @@
+import json
+from kafka import KafkaConsumer
+from typing import Optional, Dict, Union, List
+
+from .models import KafkaEvent
+from .utils import get_logger
+from .dispatcher import Dispatcher
+from .config import QueueConfig
+from .decorators import registry
+
+
+BOOTSTRAP = "localhost:9092"
+
+class Consumer:
+    def __init__( self, topics: Union[str, List[str]], config: Optional[Dict] = None, ):
+        # Convert topics to list
+        if isinstance(topics, str):
+            topics = [topics]
+
+        # Extract client overrides (only allowed fields)
+        stage = None
+        dead_letter_topic = None
+        tenant_id = None
+        source = None
+
+        if config:
+            stage = config.get("stage")
+            dead_letter_topic = config.get("dead_letter_topic")
+            tenant_id = config.get("tenant_id")
+            source = config.get("source")
+            
+
+        # SDK-controlled configuration
+        self.config = QueueConfig(
+            bootstrap_servers=BOOTSTRAP,
+            group_id=f"{tenant_id}.{source}.{stage}.group",
+            enable_auto_commit=True,
+            auto_offset_reset="earliest",
+            enable_retry=True,
+            enable_idempotency=True,
+            dead_letter_topic=dead_letter_topic,
+            stage=stage,
+        )
+
+        self.running = True
+        self.logger = get_logger(f"Consumer.{self.config.stage or 'default'}")
+
+        # Initialize Kafka Consumer
+        self.consumer = KafkaConsumer(
+            bootstrap_servers=self.config.bootstrap_servers,
+            group_id=self.config.group_id,
+            enable_auto_commit=self.config.enable_auto_commit,
+            auto_offset_reset=self.config.auto_offset_reset,
+            value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+        )
+
+        # Subscribe once
+        self.consumer.subscribe(topics)
+
+        self.dispatcher = Dispatcher(registry, self.config)
+
+        self.logger.info("Consumer initialized successfully")
+
+    def start(self):
+        self.logger.info(f"Consumer started for stage: {self.config.stage}")
+
+        try:
+            while self.running:
+                records = self.consumer.poll(timeout_ms=1000)
+
+                for tp, messages in records.items():
+                    for message in messages:
+                        try:
+                            event = KafkaEvent(**message.value)
+                            self.logger.debug(f"Processing event: {event.eventId}")
+
+                            self.dispatcher.dispatch(event)
+
+                            if not self.config.enable_auto_commit:
+                                self.consumer.commit()
+
+                        except Exception as e:
+                            event_id = message.value.get("eventId", "unknown")
+                            self.logger.error(
+                                f"Error processing event {event_id}: {e}"
+                            )
+                            continue
+
+        except KeyboardInterrupt:
+            self.logger.info("Keyboard interrupt received. Shutting down...")
+
+        except Exception as e:
+            self.logger.error(f"Unexpected error in consumer: {e}")
+
+        finally:
+            self.consumer.close()
+            self.logger.info("Consumer closed.")
+
+    def stop(self):
+        self.running = False
+        self.logger.info("Stop signal received.")

sify_queue_kafka-0.1.4/queue_kafka/decorators.py

@@ -0,0 +1,10 @@
+from .registry import EventRegistry
+
+registry = EventRegistry()
+
+
+def event_handler(event_type: str, stage: str = None):
+    def decorator(func):
+        registry.register(event_type, func, stage)
+        return func
+    return decorator

sify_queue_kafka-0.1.4/queue_kafka/dispatcher.py

@@ -0,0 +1,69 @@
+from .idempotency import IdempotencyStore
+from .retry import retry_with_backoff
+from .utils import get_logger
+from .exceptions import HandlerNotFoundError
+
+
+class Dispatcher:
+
+    def __init__(self, registry, config):
+        self.registry = registry
+        self.config = config
+        self.logger = get_logger(f"Dispatcher.{config.stage or 'default'}")
+        self.idempotency_store = IdempotencyStore()
+
+    def dispatch(self, event):
+        """Dispatch event to appropriate handler with retry and idempotency."""
+        # Check for duplicate events
+        if self.config.enable_idempotency and self.idempotency_store.is_duplicate( self.config.group_id, event.eventId ):
+            self.logger.debug(f"Duplicate event detected: {event.eventId}")
+            return
+
+        handler = self.registry.get_handler(
+            event.eventType,
+            self.config.stage
+        )
+
+        if not handler:
+            error_msg = f"No handler found for {event.eventType} @ {self.config.stage}"
+            self.logger.warning(error_msg)
+            raise HandlerNotFoundError(error_msg)
+
+        try:
+            if self.config.enable_retry:
+                retry_with_backoff(handler, event, event.maxRetries)
+            else:
+                handler(event)
+            
+            self.logger.info(f"Successfully processed event: {event.eventId}")
+            
+        except Exception as e:
+            self.logger.error(f"Failed to process event {event.eventId}: {e}")
+            
+            # Send to dead letter queue if configured
+            if self.config.dead_letter_topic:
+                self._send_to_dlq(event, str(e))
+            else:
+                raise
+
+    def _send_to_dlq(self, event, error_message):
+        """Send failed event to dead letter queue."""
+        try:
+            from .producer import Producer
+            
+            # Add error information to event metadata
+            if event.metadata is None:
+                event.metadata = {}
+            event.metadata['error'] = error_message
+            event.metadata['failed_stage'] = self.config.stage
+            event.metadata['failure_time'] = str(event.timestamp)
+            
+            producer = Producer(self.config)
+            producer.send(self.config.dead_letter_topic, event)
+            producer.close()
+            
+            self.logger.info(f"Sent event {event.eventId} to DLQ: {self.config.dead_letter_topic}")
+            
+        except Exception as dlq_error:
+            self.logger.error(f"Failed to send event {event.eventId} to DLQ: {dlq_error}")
+            raise

sify_queue_kafka-0.1.4/queue_kafka/exceptions.py

@@ -0,0 +1,33 @@
+class QueueSDKError(Exception):
+    """Base exception for all Queue SDK errors."""
+    pass
+
+
+class ConfigurationError(QueueSDKError):
+    """Raised when there's a configuration issue."""
+    pass
+
+
+class HandlerNotFoundError(QueueSDKError):
+    """Raised when no handler is found for an event type/stage combination."""
+    pass
+
+
+class EventProcessingError(QueueSDKError):
+    """Raised when event processing fails."""
+    pass
+
+
+class ProducerError(QueueSDKError):
+    """Raised when producer operations fail."""
+    pass
+
+
+class ConsumerError(QueueSDKError):
+    """Raised when consumer operations fail."""
+    pass
+
+
+class SerializationError(QueueSDKError):
+    """Raised when event serialization/deserialization fails."""
+    pass

sify_queue_kafka-0.1.4/queue_kafka/idempotency.py

@@ -0,0 +1,17 @@
+import threading
+
+
+class IdempotencyStore:
+    def __init__(self):
+        self.processed_ids = set()
+        self.lock = threading.Lock()
+
+    def is_duplicate(self, group_id: str, event_id: str) -> bool:
+        key = f"{group_id}:{event_id}"
+
+        with self.lock:
+            if key in self.processed_ids:
+                return True
+
+            self.processed_ids.add(key)
+            return False

sify_queue_kafka-0.1.4/queue_kafka/models.py

@@ -0,0 +1,23 @@
+from pydantic import BaseModel, Field
+from typing import Any, Dict, Optional
+from datetime import datetime
+import uuid
+
+
+class KafkaEvent(BaseModel):
+    eventId: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    eventType: str
+    eventVersion: str
+    source: str
+    tenantId: str
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+    correlationId: Optional[str] = None
+    traceId: Optional[str] = None
+
+    priority: Optional[str] = "MEDIUM"
+    retryCount: int = 0
+    maxRetries: int = 5
+
+    payload: Dict[str, Any]
+    metadata: Optional[Dict[str, Any]] = None

sify_queue_kafka-0.1.4/queue_kafka/producer.py

@@ -0,0 +1,74 @@
+import json
+import uuid
+from datetime import datetime, timezone
+from kafka import KafkaProducer
+from .models import KafkaEvent
+from .utils import get_logger
+from .exceptions import ProducerError
+from typing import Union, List
+
+
+class Producer:
+    def __init__(self):
+        self.logger = get_logger("Producer")
+        
+        try:
+            self.producer = KafkaProducer(
+                bootstrap_servers="localhost:9092",
+                value_serializer=lambda v: json.dumps(v, default=str).encode("utf-8"),
+                key_serializer=lambda k: k.encode("utf-8") if k else None,
+                acks="all",
+                retries=5,
+                enable_idempotence=True
+            )
+            self.logger.info("Producer initialized successfully")
+        except Exception as e:
+            self.logger.error(f"Failed to initialize producer: {e}")
+            raise ProducerError(f"Producer initialization failed: {e}")
+
+    def send(self, topic: Union[str, List[str]], message: dict) -> None:
+        try:
+            # Create KafkaEvent internally
+            event = KafkaEvent(
+                eventId=str(uuid.uuid4()),
+                eventType=message.get("eventType"),
+                eventVersion=message.get("eventVersion", "1.0"),
+                source=message.get("source"),
+                tenantId=message.get("tenantId"),
+                timestamp=datetime.now(timezone.utc),
+                correlationId=str(uuid.uuid4()),
+                traceId=str(uuid.uuid4()),
+                priority=message.get("priority", "MEDIUM"),
+                payload=message.get("payload", {}),
+                metadata=message.get("metadata", {})
+            )
+
+            event_dict = event.model_dump(mode="json")
+
+            topics = [topic] if isinstance(topic, str) else topic
+
+            for t in topics:
+                self.producer.send(t, value=event_dict, key=event.eventId)
+
+        except Exception as e:
+            self.logger.error(f"Failed to send event: {e}")
+            raise ProducerError(f"Failed to send event: {e}")
+
+    def flush(self):
+        """Flush pending messages."""
+        try:
+            self.producer.flush()
+            self.logger.debug("Producer flushed successfully")
+        except Exception as e:
+            self.logger.error(f"Failed to flush producer: {e}")
+            raise ProducerError(f"Failed to flush producer: {e}")
+
+    def close(self):
+        """Close producer and clean up resources."""
+        try:
+            self.producer.flush()
+            self.producer.close()
+            self.logger.info("Producer closed successfully")
+        except Exception as e:
+            self.logger.error(f"Error closing producer: {e}")
+            raise ProducerError(f"Error closing producer: {e}")

sify_queue_kafka-0.1.4/queue_kafka/registry.py

@@ -0,0 +1,13 @@
+class EventRegistry:
+    def __init__(self):
+        self._handlers = {}
+
+    def register(self, event_type, handler, stage=None):
+        key = (event_type, stage)
+        self._handlers[key] = handler
+
+    def get_handler(self, event_type, stage=None):
+        return self._handlers.get((event_type, stage))
+
+
+registry = EventRegistry()

sify_queue_kafka-0.1.4/queue_kafka/retry.py

@@ -0,0 +1,45 @@
+import time
+from typing import Callable, Any
+from .utils import get_logger
+
+
+def retry_with_backoff(handler: Callable[[Any], Any], event: Any, max_retries: int, 
+                      base_delay: float = 1.0, max_delay: float = 60.0) -> Any:
+    """
+    Retry handler with exponential backoff.
+    
+    Args:
+        handler: The function to retry
+        event: The event to pass to the handler
+        max_retries: Maximum number of retry attempts
+        base_delay: Initial delay between retries (seconds)
+        max_delay: Maximum delay between retries (seconds)
+    
+    Returns:
+        Result from handler function
+    
+    Raises:
+        MaxRetriesExceededError: If all retries are exhausted
+    """
+    logger = get_logger("Retry")
+    
+    while event.retryCount < max_retries:
+        try:
+            return handler(event)
+        except Exception as e:
+            event.retryCount += 1
+            
+            if event.retryCount >= max_retries:
+                logger.error(f"Max retries ({max_retries}) exceeded for event {event.eventId}")
+                raise MaxRetriesExceededError(f"Failed after {max_retries} retries: {e}")
+            
+            # Calculate exponential backoff with jitter
+            delay = min(base_delay * (2 ** (event.retryCount - 1)), max_delay)
+            jitter = delay * 0.1 * (0.5 - time.time() % 1)  # ±10% jitter
+            final_delay = delay + jitter
+            
+            logger.warning(f"Retry {event.retryCount}/{max_retries} for event {event.eventId} "
+                          f"after {final_delay:.2f}s. Error: {e}")
+            time.sleep(final_delay)
+    
+    raise MaxRetriesExceededError(f"Failed after {max_retries} retries")

sify_queue_kafka-0.1.4/queue_kafka/topic_resolver.py

@@ -0,0 +1,6 @@
+class TopicResolver:
+    def __init__(self, environment: str):
+        self.environment = environment
+
+    def resolve(self, logical_topic: str) -> str:
+        return f"{self.environment}.{logical_topic}"

sify_queue_kafka-0.1.4/queue_kafka/utils.py

@@ -0,0 +1,22 @@
+import logging
+from typing import Optional
+
+
+def get_logger(name: str, level: Optional[int] = None) -> logging.Logger:
+    """Get a configured logger with consistent formatting."""
+    logger = logging.getLogger(name)
+    
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    
+    if level:
+        logger.setLevel(level)
+    elif logger.level == logging.NOTSET:
+        logger.setLevel(logging.INFO)
+    
+    return logger

{sify_queue_kafka-0.1.3 → sify_queue_kafka-0.1.4}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sify-queue-kafka
-version = 0.1.3
+version = 0.1.4
 description = Enterprise Queue management Framework
 long_description = file: README.md
 long_description_content_type = text/markdown

{sify_queue_kafka-0.1.3 → sify_queue_kafka-0.1.4}/sify_queue_kafka.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sify-queue-kafka
-Version: 0.1.3
+Version: 0.1.4
 Summary: Enterprise Queue management Framework
 Author: sifymodernization
 Author-email: sifymodernization <sifymodernization.dev@sifycorp.com>
@@ -192,32 +192,33 @@ if __name__ == "__main__":
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `bootstrap_servers` | str | Required | Kafka bootstrap servers |
-| `group_id` | str | None | Consumer group ID |
+| `bootstrap_servers` | str | "localhost:9092" | Kafka bootstrap servers |
+| `group_id` | str | "" | Consumer group ID. Is generated with the tenant_id and source from the consumer config |
 | `enable_auto_commit` | bool | False | Enable auto commit of offsets |
 | `auto_offset_reset` | str | "earliest" | Offset reset policy |
 | `enable_idempotency` | bool | True | Enable duplicate detection |
 | `enable_retry` | bool | True | Enable retry with backoff |
 | `dead_letter_topic` | str | None | Topic for failed events |
-| `stage` | str | None | Processing stage name |
+| `stage` | str | "" | Processing stage name |
+| `enable_idempotent_producer` | bool | False | Enable idempotent producer |
 
 ## Event Model
 
 ```python
 class KafkaEvent(BaseModel):
-    eventId: str 
+    eventId: str = Field(default_factory=lambda: str(uuid.uuid4()))
     eventType: str
     eventVersion: str
     source: str
     tenantId: str
-    timestamp: datetime
-    correlationId: Optional[str]
-    traceId: Optional[str]
-    priority: Optional[str]
-    retryCount: int
-    maxRetries: int
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    correlationId: Optional[str] = None
+    traceId: Optional[str] = None
+    priority: Optional[str] = "MEDIUM"
+    retryCount: int = 0
+    maxRetries: int = 5
     payload: Dict[str, Any]
-    metadata: Optional[Dict[str, Any]]
+    metadata: Optional[Dict[str, Any]] = None
 ```
 
 ## Exception Types
@@ -260,7 +261,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 ## Changelog
 
-### v0.1.3
+### v0.1.4
 - Initial release
 - Basic producer/consumer functionality
 - Event validation with Pydantic