fastsqs 0.3.2__tar.gz → 0.3.4__tar.gz

{fastsqs-0.3.2/fastsqs.egg-info → fastsqs-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastsqs
-Version: 0.3.2
+Version: 0.3.4
 Summary: FastAPI-like, modern async SQS message processing for Python with advanced features
 Home-page: https://github.com/lafayettegabe/fastsqs
 Author: Gabriel LaFayette
@@ -27,9 +27,21 @@ Requires-Dist: pydantic>=2.0.0
 Provides-Extra: dynamodb
 Requires-Dist: boto3>=1.26.0; extra == "dynamodb"
 Requires-Dist: aioboto3>=12.0.0; extra == "dynamodb"
+Provides-Extra: idempotency
+Requires-Dist: boto3>=1.26.0; extra == "idempotency"
+Requires-Dist: aioboto3>=12.0.0; extra == "idempotency"
+Provides-Extra: telemetry
+Requires-Dist: opentelemetry-sdk>=1.32.1; extra == "telemetry"
+Requires-Dist: opentelemetry-exporter-otlp>=1.32.1; extra == "telemetry"
+Requires-Dist: opentelemetry-instrumentation>=0.53b1; extra == "telemetry"
+Requires-Dist: elasticsearch>=8.11.1; extra == "telemetry"
 Provides-Extra: all
 Requires-Dist: boto3>=1.26.0; extra == "all"
 Requires-Dist: aioboto3>=12.0.0; extra == "all"
+Requires-Dist: opentelemetry-sdk>=1.32.1; extra == "all"
+Requires-Dist: opentelemetry-exporter-otlp>=1.32.1; extra == "all"
+Requires-Dist: opentelemetry-instrumentation>=0.53b1; extra == "all"
+Requires-Dist: elasticsearch>=8.11.1; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{fastsqs-0.3.2 → fastsqs-0.3.4}/fastsqs/app.py

@@ -10,6 +10,7 @@ from .types import QueueType, Handler
 from .exceptions import RouteNotFound, InvalidMessage
 from .middleware import Middleware, run_middlewares
 from .middleware.idempotency import IdempotencyHit
+from .middleware.logging import LoggingMiddleware
 from .routing import QueueRouter
 from .utils import group_records_by_message_group, invoke_handler
 from .presets import MiddlewarePreset
@@ -66,7 +67,7 @@ class FastSQS:
                     if variant not in self._route_lookup:
                         self._route_lookup[variant] = primary_type
                     elif self.debug:
-                        print(f"Warning: Message type variant '{variant}' conflicts with existing route")
+                        self._log("warning", f"Message type variant conflicts with existing route", variant=variant)
             
             return handler
         
@@ -92,7 +93,17 @@ class FastSQS:
         self._routers.append(router)
 
     def add_middleware(self, middleware: Middleware) -> None:
+        middleware._app = self
         self._middlewares.append(middleware)
+    
+    def use(self, middleware: Middleware) -> None:
+        self.add_middleware(middleware)
+
+    def _log(self, level: str, message: str, **data) -> None:
+        for middleware in self._middlewares:
+            if isinstance(middleware, LoggingMiddleware) and hasattr(middleware, 'log'):
+                middleware.log(level, message, **data)
+                return
 
     def use_preset(self, preset: str, **kwargs) -> None:
         if preset == "production":
@@ -110,7 +121,7 @@ class FastSQS:
     def set_queue_type(self, queue_type: QueueType) -> None:
         self.queue_type = queue_type
         if self.debug:
-            print(f"[FastSQS] Queue type set to: {queue_type.value}")
+            self._log("info", f"Queue type set to: {queue_type.value}")
 
     def is_fifo_queue(self) -> bool:
         return self.queue_type == QueueType.FIFO
@@ -119,11 +130,16 @@ class FastSQS:
         body_str = record.get("body", "")
         msg_id = record.get("messageId") or record.get("message_id") or "UNKNOWN"
 
+        self._log("info", f"Starting record processing", msg_id=msg_id)
+        self._log("debug", f"Raw body", msg_id=msg_id, body=body_str[:500] + ('...' if len(body_str) > 500 else ''))
+
         try:
             payload = json.loads(body_str) if body_str else {}
             if not isinstance(payload, dict):
                 raise InvalidMessage("Message body must be a JSON object")
+            self._log("debug", f"Parsed payload", msg_id=msg_id, payload=payload)
         except json.JSONDecodeError as e:
+            self._log("error", f"JSON decode error", msg_id=msg_id, error=str(e))
             raise InvalidMessage(f"Invalid JSON in message body: {e}")
 
         ctx: Dict[str, Any] = {
@@ -141,15 +157,19 @@ class FastSQS:
                 "messageDeduplicationId": attributes.get("messageDeduplicationId"),
                 "queueType": "fifo"
             }
+            self._log("debug", f"FIFO info", msg_id=msg_id, fifo_info=ctx["fifoInfo"])
 
         err: Optional[Exception] = None
         result: Any = None
         
         try:
+            self._log("debug", f"Running 'before' middleware chain", msg_id=msg_id)
             await run_middlewares(self._middlewares, "before", payload, record, context, ctx)
+            self._log("debug", f"'before' middleware chain completed", msg_id=msg_id)
         except IdempotencyHit as idempotency_hit:
+            self._log("info", f"Idempotency hit, returning cached result", msg_id=msg_id, key=idempotency_hit.key)
             if self.debug:
-                print(f"[FastSQS] Idempotency hit for message {msg_id}: {idempotency_hit.key}")
+                self._log("debug", f"Idempotency hit for message", msg_id=msg_id, key=idempotency_hit.key)
             ctx["idempotency_result"] = idempotency_hit.result
             ctx["idempotency_hit"] = True
             await run_middlewares(self._middlewares, "after", payload, record, context, ctx, None)
@@ -159,47 +179,66 @@ class FastSQS:
             handled = False
             
             message_type = payload.get(self.message_type_key)
+            self._log("debug", f"Message type", msg_id=msg_id, message_type=message_type)
+            
             if message_type:
                 route = self._find_route(message_type)
                 if route:
                     event_model, handler = route
+                    self._log("debug", f"Found route for {message_type}", msg_id=msg_id, handler=handler.__name__)
                     
                     try:
                         event_instance = event_model.model_validate(payload)
+                        self._log("debug", f"Payload validation successful", msg_id=msg_id)
                     except ValidationError as e:
+                        self._log("error", f"Validation failed", msg_id=msg_id, error=str(e))
                         raise InvalidMessage(f"Validation failed for {message_type}: {e}")
 
                     ctx["message_type"] = message_type
+                    self._log("debug", f"Invoking handler", msg_id=msg_id, handler=handler.__name__)
                     result = await invoke_handler(handler, msg=event_instance, record=record, context=context, ctx=ctx)
+                    self._log("debug", f"Handler completed successfully", msg_id=msg_id, result_type=type(result).__name__)
                     ctx["handler_result"] = result
                     handled = True
+                else:
+                    self._log("warning", f"No route found for message type", msg_id=msg_id, message_type=message_type)
 
             if not handled and self._routers:
-                for router in self._routers:
+                self._log("debug", f"Trying routers", msg_id=msg_id, router_count=len(self._routers))
+                for i, router in enumerate(self._routers):
+                    self._log("debug", f"Trying router {i}", msg_id=msg_id, router_key=router.key)
                     if await router.dispatch(payload, record, context, ctx, root_payload=payload):
+                        self._log("debug", f"Router {i} handled the message", msg_id=msg_id)
                         handled = True
                         result = ctx.get("handler_result")
                         break
+                    else:
+                        self._log("debug", f"Router {i} did not handle the message", msg_id=msg_id)
 
             if not handled:
                 if self._default_handler:
+                    self._log("debug", f"Using default handler", msg_id=msg_id)
                     result = await invoke_handler(self._default_handler, payload=payload, record=record, context=context, ctx=ctx)
                     ctx["handler_result"] = result
                 else:
                     available_routes = list(self._routes.keys())
                     available_routers = [r.key for r in self._routers]
-                    raise RouteNotFound(
-                        f"No handler found for message. "
-                        f"Available FastSQS routes: {available_routes}, "
-                        f"Available router keys: {available_routers}"
-                    )
+                    error_msg = (f"No handler found for message. "
+                               f"Available FastSQS routes: {available_routes}, "
+                               f"Available router keys: {available_routers}")
+                    self._log("error", error_msg, msg_id=msg_id, available_routes=available_routes, available_routers=available_routers)
+                    raise RouteNotFound(error_msg)
                     
         except Exception as e:
+            self._log("error", f"Handler error", msg_id=msg_id, error_type=type(e).__name__, error=str(e))
             err = e
             raise
         finally:
+            self._log("debug", f"Running 'after' middleware chain", msg_id=msg_id)
             await run_middlewares(self._middlewares, "after", payload, record, context, ctx, err)
+            self._log("debug", f"'after' middleware chain completed", msg_id=msg_id)
         
+        self._log("info", f"Record processing completed successfully", msg_id=msg_id)
         return result
 
     async def _handle_event(self, event: dict, context: Any) -> dict:
@@ -209,7 +248,7 @@ class FastSQS:
         
         if self.debug:
             queue_info = f"queue_type={self.queue_type.value}, records={len(records)}"
-            print(f"[FastSQS] Processing event: {queue_info}")
+            self._log("info", f"Processing event", queue_info=queue_info)
         
         if self.is_fifo_queue():
             return await self._handle_fifo_event(records, context)
@@ -219,6 +258,8 @@ class FastSQS:
     async def _handle_standard_event(self, records: List[dict], context: Any) -> dict:
         failures: List[Dict[str, str]] = []
         
+        self._log("info", f"Processing records in standard queue mode", record_count=len(records))
+        
         semaphore = asyncio.Semaphore(self.max_concurrent_messages)
         
         async def process_with_semaphore(record):
@@ -231,11 +272,16 @@ class FastSQS:
         for i, result in enumerate(results):
             if isinstance(result, Exception):
                 msg_id = records[i].get("messageId", "UNKNOWN")
+                self._log("error", f"Record failed", msg_id=msg_id, error_type=type(result).__name__, error=str(result))
                 if self.debug:
-                    print(f"[FastSQS] Record failed: messageId={msg_id}, error={result}")
+                    self._log("debug", f"Record failed", msg_id=msg_id, error=str(result))
                 if self.enable_partial_batch_failure:
                     failures.append({"itemIdentifier": msg_id})
+            else:
+                msg_id = records[i].get("messageId", "UNKNOWN")
+                self._log("debug", f"Record succeeded", msg_id=msg_id)
         
+        self._log("info", f"Batch processing completed", succeeded=len(records) - len(failures), failed=len(failures))
         return {"batchItemFailures": failures}
     
     async def _handle_fifo_event(self, records: List[dict], context: Any) -> dict:
@@ -244,12 +290,12 @@ class FastSQS:
         message_groups = group_records_by_message_group(records)
         
         if self.debug:
-            print(f"[FastSQS] FIFO processing: {len(records)} records in {len(message_groups)} groups")
+            self._log("info", f"FIFO processing", record_count=len(records), group_count=len(message_groups))
         
         async def process_group(group_id: str, group_records: List[dict]):
             group_failures = []
             if self.debug:
-                print(f"[FastSQS] Processing group '{group_id}' with {len(group_records)} records")
+                self._log("debug", f"Processing group", group_id=group_id, record_count=len(group_records))
             
             for rec in group_records:
                 try:
@@ -257,7 +303,7 @@ class FastSQS:
                 except Exception as e:
                     msg_id = rec.get("messageId", "UNKNOWN")
                     if self.debug:
-                        print(f"[FastSQS] FIFO record failed: messageId={msg_id}, group={group_id}, error={e}")
+                        self._log("error", f"FIFO record failed", msg_id=msg_id, group_id=group_id, error=str(e))
                     if self.enable_partial_batch_failure:
                         group_failures.append({"itemIdentifier": msg_id})
             
@@ -275,12 +321,17 @@ class FastSQS:
                 failures.extend(result)
             elif isinstance(result, Exception):
                 if self.debug:
-                    print(f"[FastSQS] Message group processing failed: {result}")
+                    self._log("error", f"Message group processing failed", error=str(result))
         
         return {"batchItemFailures": failures}
     
     async def _handle_record_safe(self, record: dict, context: Any) -> None:
-        await self._handle_record(record, context)
+        msg_id = record.get("messageId", "UNKNOWN")
+        try:
+            await self._handle_record(record, context)
+        except Exception as e:
+            self._log("error", f"Record processing failed", msg_id=msg_id, error_type=type(e).__name__, error=str(e))
+            raise
 
     def handler(self, event: dict, context: Any) -> dict:
         try:

fastsqs-0.3.4/fastsqs/concurrency/__init__.py

@@ -0,0 +1,2 @@
+from .concurrency import ThreadPoolManager
+from .decorators import background

fastsqs-0.3.4/fastsqs/concurrency/concurrency.py

@@ -0,0 +1,36 @@
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+
+
+class ThreadPoolManager:
+    """
+    Manager for a shared ThreadPoolExecutor with a hard cap on the
+    number of worker threads (max = 1024 for AWS Lambda safety).
+    """
+
+    MAX_WORKERS: int = 1024
+    THREAD_NAME_PREFIX: str = "thread-pool-manager"
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._configure_executor()
+        return cls._instance
+
+    def _configure_executor(self) -> None:
+        self._executor = ThreadPoolExecutor(
+            max_workers=self.MAX_WORKERS,
+            thread_name_prefix=self.THREAD_NAME_PREFIX,
+        )
+
+    @staticmethod
+    def _swallow(call: Callable[[], None]) -> None:
+        try:
+            call()
+        except Exception:
+            pass
+
+    def submit(self, fn: Callable, *args, **kwargs) -> None:
+        """Submit a task to the shared executor (fire-and-forget), swallowing exceptions."""
+        self._executor.submit(self._swallow, lambda: fn(*args, **kwargs))

fastsqs-0.3.4/fastsqs/concurrency/decorators.py

@@ -0,0 +1,20 @@
+import functools
+from typing import Callable
+
+from .concurrency import ThreadPoolManager
+
+
+def background(func: Callable):
+    """
+    Decorator to execute the function in the background using the shared thread pool.
+    Exceptions inside the function are swallowed by the manager. Returns None immediately.
+
+    Usage:
+        @background
+        def my_task(...):
+            ...
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        ThreadPoolManager().submit(func, *args, **kwargs)
+    return wrapper

fastsqs-0.3.4/fastsqs/concurrency.py

@@ -0,0 +1,16 @@
+import functools
+import asyncio
+from typing import Any, Callable
+
+
+def background(func: Callable) -> Callable:
+    if asyncio.iscoroutinefunction(func):
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            return await func(*args, **kwargs)
+        return async_wrapper
+    else:
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+        return sync_wrapper

fastsqs-0.3.4/fastsqs/logger/__init__.py

@@ -0,0 +1 @@
+from .logger import Logger

fastsqs-0.3.4/fastsqs/logger/config.py

@@ -0,0 +1,12 @@
+import os
+
+
+class OtelConfig:
+    def __init__(self):
+        self.use_otel = os.getenv("USE_OTEL", "true").lower() != "false"
+        self.environment = os.getenv("OTEL_ENVIRONMENT", "production")
+        self.service_name = os.getenv("OTEL_SERVICE_NAME", "fastsqs-service")
+        self.tracer_name = os.getenv("OTEL_TRACER_NAME", "fastsqs-tracer")
+        self.elastic_url = os.getenv("OTEL_ELASTIC_URL", None)
+        self.elastic_token = os.getenv("OTEL_ELASTIC_TOKEN", None)
+        self.exporter_endpoint = os.getenv("OTEL_EXPORTER_ENDPOINT", "http://localhost:4318/v1/traces")

fastsqs-0.3.4/fastsqs/logger/elasticsearch_handler.py

@@ -0,0 +1,56 @@
+import logging
+from enum import Enum
+
+from .config import OtelConfig
+from .utils import get_elastic_client, build_log_entry
+
+
+class Environment(Enum):
+    PRODUCTION = "production"
+    HOMOLOG = "homolog"
+    LOCAL = "local"
+
+
+otel_config = OtelConfig()
+OTEL_ENVIRONMENT = otel_config.environment
+OTEL_SERVICE_NAME = otel_config.service_name
+USE_OTEL = otel_config.use_otel
+
+
+class ElasticsearchHandler(logging.Handler):
+    ELASTICSEARCH_ENVIRONMENTS = {
+        Environment.PRODUCTION.value,
+        Environment.HOMOLOG.value
+    }
+    
+    def __init__(self, index_name=None, use_otel=USE_OTEL):
+        super().__init__()
+
+        self.enable_logging = use_otel and OTEL_ENVIRONMENT in self.ELASTICSEARCH_ENVIRONMENTS
+
+        if self.enable_logging:
+            self.es_client = get_elastic_client()
+            self.index_name = (
+                index_name or
+                f"logs-{OTEL_SERVICE_NAME}-{OTEL_ENVIRONMENT}"
+            )
+
+    def emit(self, record):
+        try:
+            if not self.enable_logging:
+                return
+
+            log_entry = build_log_entry(
+                record=record,
+                service_name=OTEL_SERVICE_NAME,
+                environment=OTEL_ENVIRONMENT,
+                formatter=self
+            )
+
+            if self.es_client:
+                self.es_client.index(index=self.index_name, document=log_entry)
+
+        except Exception as e:
+            logging.getLogger(__name__).error(
+                f"Failed to send log to Elasticsearch: {e}"
+            )

fastsqs-0.3.4/fastsqs/logger/logger.py

@@ -0,0 +1,53 @@
+import logging
+import json
+import os
+
+from .config import OtelConfig
+from ..concurrency.decorators import background
+from .elasticsearch_handler import ElasticsearchHandler
+
+otel_config = OtelConfig()
+USE_OTEL = otel_config.use_otel
+
+class Logger:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(Logger, cls).__new__(cls)
+            cls._instance._configure_logger()
+        return cls._instance
+
+    def _configure_logger(self):
+        logging.getLogger().setLevel(logging.WARNING)
+        self.use_otel = USE_OTEL
+        self._logger = logging.getLogger("appLogger")
+        self._logger.setLevel(logging.INFO)
+        self._logger.propagate = False
+        if not self._logger.handlers:
+            self._add_handler(logging.StreamHandler())
+            if USE_OTEL:
+                try:
+                    self._add_handler(ElasticsearchHandler(use_otel=self.use_otel))
+                except Exception as e:
+                    self._logger.warning(f"Failed to initialize ElasticsearchHandler: {e}")
+
+    def _add_handler(self, handler):
+        try:
+            formatter = logging.Formatter("%(message)s")
+            handler.setFormatter(formatter)
+            self._logger.addHandler(handler)
+        except Exception as e:
+            self._logger.error(
+                f"Failed to configure {handler.__class__.__name__}: {e}"
+            )
+
+    @background
+    def log(self, level, message, **data):
+        log_method = getattr(self._logger, level, self._logger.debug)
+        if data:
+            try:
+                message = f"{message} | data: {json.dumps(data, default=str)}"
+            except Exception:
+                message = f"{message} | data: {data}"
+        log_method(message)

fastsqs-0.3.4/fastsqs/logger/utils.py

@@ -0,0 +1,41 @@
+from datetime import datetime, timezone
+from elasticsearch import Elasticsearch
+from .config import OtelConfig
+
+otel_config = OtelConfig()
+ELASTIC_URL = otel_config.elastic_url
+ELASTIC_TOKEN = otel_config.elastic_token
+
+_es_client = None
+
+
+def filter_extra_fields(record_dict):
+    allowed_fields = {"data"}
+    return {key: value for key, value in record_dict.items() if key in allowed_fields}
+
+
+def get_elastic_client():
+    global _es_client
+    if _es_client is None:
+        _es_client = Elasticsearch(
+            ELASTIC_URL, 
+            api_key=ELASTIC_TOKEN,
+            request_timeout=10
+        )
+    return _es_client
+
+
+def build_log_entry(record, request_id, service_name, environment, formatter):
+    return {
+        "@timestamp": datetime.now(timezone.utc).isoformat(),
+        "log.level": record.levelname.lower(),
+        "message": formatter.format(record),
+        "app.name": service_name,
+        "x-request-id": request_id,
+        "service.name": service_name,
+        "environment": environment,
+        "extra": filter_extra_fields(record.__dict__),
+        "agent": {
+            "name": "opentelemetry/python",
+        },
+    }

{fastsqs-0.3.2 → fastsqs-0.3.4}/fastsqs/middleware/base.py

@@ -5,6 +5,15 @@ from typing import Any, Awaitable, List, Optional
 
 
 class Middleware:
+    def __init__(self):
+        self._app = None  # Will be set by FastSQS when middleware is added
+    
+    def _log(self, level: str, message: str, **data) -> None:
+        """Log method that routes through the app's logging system"""
+        if self._app and hasattr(self._app, '_log'):
+            self._app._log(level, message, **data)
+        # If no app or logging available, silently do nothing
+    
     async def before(self, payload: dict, record: dict, context: Any, ctx: dict) -> None:
         return None