dory-processor-sdk 0.0.1__py3-none-any.whl

dory/monitoring/opentelemetry.py

@@ -0,0 +1,489 @@
+"""
+OpenTelemetry Integration
+
+Provides full OpenTelemetry support for distributed tracing and metrics.
+
+Features:
+- Automatic span creation
+- Trace propagation
+- Metrics collection
+- Context management
+- Integration with request IDs
+
+Note: Requires opentelemetry packages:
+    pip install opentelemetry-api opentelemetry-sdk
+    pip install opentelemetry-exporter-otlp
+"""
+
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+from functools import wraps
+from typing import Any, Dict, Optional, Callable
+
+logger = logging.getLogger(__name__)
+
+# Try to import OpenTelemetry (optional dependency)
+try:
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import (
+        BatchSpanProcessor,
+        ConsoleSpanExporter,
+    )
+    from opentelemetry.sdk.resources import Resource, SERVICE_NAME
+    from opentelemetry.trace import Status, StatusCode, SpanKind
+    from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+    OTEL_AVAILABLE = True
+except ImportError:
+    OTEL_AVAILABLE = False
+    # Create stub types for when OpenTelemetry is not available
+    # This allows the module to be imported without OpenTelemetry installed
+    from enum import Enum
+
+    class SpanKind(Enum):
+        """Stub SpanKind enum when OpenTelemetry is not installed."""
+        INTERNAL = 0
+        SERVER = 1
+        CLIENT = 2
+        PRODUCER = 3
+        CONSUMER = 4
+
+    trace = None
+    TracerProvider = None
+
+    logger.warning(
+        "OpenTelemetry not available. Install with: "
+        "pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp"
+    )
+
+
+class OpenTelemetryManager:
+    """
+    Manages OpenTelemetry tracing and metrics.
+
+    Features:
+    - Automatic tracer setup
+    - Span creation and management
+    - Context propagation
+    - Metrics collection
+    - Export to OTLP endpoints
+
+    Usage:
+        # Initialize
+        otel = OpenTelemetryManager(service_name="my-service")
+        otel.initialize()
+
+        # Use decorator
+        @otel.trace("process_item")
+        async def process_item(item):
+            # Automatically traced
+            pass
+
+        # Or context manager
+        with otel.create_span("operation"):
+            # Operations here are traced
+            pass
+    """
+
+    def __init__(
+        self,
+        service_name: str = "dory-processor",
+        service_version: str = "1.0.0",
+        environment: str = "production",
+        otlp_endpoint: Optional[str] = None,
+        console_export: bool = False,
+        enable_metrics: bool = True,
+    ):
+        """
+        Initialize OpenTelemetry manager.
+
+        Args:
+            service_name: Name of the service
+            service_version: Version of the service
+            environment: Environment (production, staging, dev)
+            otlp_endpoint: Optional OTLP endpoint URL
+            console_export: Export spans to console (for debugging)
+            enable_metrics: Enable metrics collection
+        """
+        if not OTEL_AVAILABLE:
+            raise ImportError(
+                "OpenTelemetry not available. Install with: "
+                "pip install opentelemetry-api opentelemetry-sdk"
+            )
+
+        self.service_name = service_name
+        self.service_version = service_version
+        self.environment = environment
+        self.otlp_endpoint = otlp_endpoint
+        self.console_export = console_export
+        self.enable_metrics = enable_metrics
+
+        self._tracer_provider: Optional[TracerProvider] = None
+        self._tracer: Optional[trace.Tracer] = None
+        self._initialized = False
+
+        logger.info(
+            f"OpenTelemetryManager created: service={service_name}, "
+            f"version={service_version}, env={environment}"
+        )
+
+    def initialize(self) -> None:
+        """
+        Initialize OpenTelemetry.
+
+        Sets up tracer provider, exporters, and processors.
+        """
+        if self._initialized:
+            logger.warning("OpenTelemetry already initialized")
+            return
+
+        # Create resource
+        resource = Resource.create({
+            SERVICE_NAME: self.service_name,
+            "service.version": self.service_version,
+            "deployment.environment": self.environment,
+        })
+
+        # Create tracer provider
+        self._tracer_provider = TracerProvider(resource=resource)
+
+        # Add console exporter if enabled
+        if self.console_export:
+            console_exporter = ConsoleSpanExporter()
+            console_processor = BatchSpanProcessor(console_exporter)
+            self._tracer_provider.add_span_processor(console_processor)
+            logger.info("Console span exporter added")
+
+        # Add OTLP exporter if endpoint provided
+        if self.otlp_endpoint:
+            try:
+                from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+                otlp_exporter = OTLPSpanExporter(endpoint=self.otlp_endpoint)
+                otlp_processor = BatchSpanProcessor(otlp_exporter)
+                self._tracer_provider.add_span_processor(otlp_processor)
+                logger.info(f"OTLP span exporter added: {self.otlp_endpoint}")
+            except ImportError:
+                logger.warning(
+                    "OTLP exporter not available. Install with: "
+                    "pip install opentelemetry-exporter-otlp"
+                )
+
+        # Set global tracer provider
+        trace.set_tracer_provider(self._tracer_provider)
+
+        # Get tracer
+        self._tracer = trace.get_tracer(
+            instrumenting_module_name=__name__,
+            instrumenting_library_version=self.service_version,
+        )
+
+        self._initialized = True
+        logger.info("OpenTelemetry initialized successfully")
+
+    def get_tracer(self) -> trace.Tracer:
+        """
+        Get the tracer instance.
+
+        Returns:
+            OpenTelemetry tracer
+        """
+        if not self._initialized:
+            self.initialize()
+        return self._tracer
+
+    @contextmanager
+    def create_span(
+        self,
+        name: str,
+        attributes: Optional[Dict[str, Any]] = None,
+        kind: SpanKind = SpanKind.INTERNAL,
+    ):
+        """
+        Create a span context manager.
+
+        Args:
+            name: Span name
+            attributes: Optional span attributes
+            kind: Span kind (INTERNAL, SERVER, CLIENT, etc.)
+
+        Example:
+            with otel.create_span("database_query", {"query": "SELECT ...")):
+                result = await db.execute(query)
+        """
+        if not self._initialized:
+            # If not initialized, just yield without tracing
+            yield None
+            return
+
+        tracer = self.get_tracer()
+
+        with tracer.start_as_current_span(name, kind=kind) as span:
+            # Add attributes
+            if attributes:
+                for key, value in attributes.items():
+                    span.set_attribute(key, value)
+
+            try:
+                yield span
+            except Exception as e:
+                # Record exception
+                span.record_exception(e)
+                span.set_status(Status(StatusCode.ERROR, str(e)))
+                raise
+            else:
+                span.set_status(Status(StatusCode.OK))
+
+    def trace(
+        self,
+        name: Optional[str] = None,
+        attributes: Optional[Dict[str, Any]] = None,
+        kind: SpanKind = SpanKind.INTERNAL,
+    ):
+        """
+        Decorator to automatically trace a function.
+
+        Args:
+            name: Span name (uses function name if None)
+            attributes: Optional span attributes
+            kind: Span kind
+
+        Example:
+            @otel.trace("process_item")
+            async def process_item(item):
+                # Automatically traced
+                pass
+        """
+        def decorator(func):
+            span_name = name or func.__name__
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                with self.create_span(span_name, attributes, kind):
+                    return await func(*args, **kwargs)
+
+            @wraps(func)
+            def sync_wrapper(*args, **kwargs):
+                with self.create_span(span_name, attributes, kind):
+                    return func(*args, **kwargs)
+
+            # Return appropriate wrapper
+            import asyncio
+            if asyncio.iscoroutinefunction(func):
+                return async_wrapper
+            else:
+                return sync_wrapper
+
+        return decorator
+
+    def add_span_attributes(self, attributes: Dict[str, Any]) -> None:
+        """
+        Add attributes to current span.
+
+        Args:
+            attributes: Attributes to add
+        """
+        if not self._initialized:
+            return
+
+        span = trace.get_current_span()
+        if span:
+            for key, value in attributes.items():
+                span.set_attribute(key, value)
+
+    def record_exception(self, exception: Exception) -> None:
+        """
+        Record an exception in current span.
+
+        Args:
+            exception: Exception to record
+        """
+        if not self._initialized:
+            return
+
+        span = trace.get_current_span()
+        if span:
+            span.record_exception(exception)
+            span.set_status(Status(StatusCode.ERROR, str(exception)))
+
+    def inject_context(self, headers: Dict[str, str]) -> Dict[str, str]:
+        """
+        Inject trace context into headers for propagation.
+
+        Args:
+            headers: HTTP headers or similar dict
+
+        Returns:
+            Headers with trace context injected
+        """
+        if not self._initialized:
+            return headers
+
+        propagator = TraceContextTextMapPropagator()
+        headers = dict(headers)
+        propagator.inject(headers)
+        return headers
+
+    def extract_context(self, headers: Dict[str, str]) -> None:
+        """
+        Extract trace context from headers.
+
+        Args:
+            headers: HTTP headers or similar dict
+        """
+        if not self._initialized:
+            return
+
+        propagator = TraceContextTextMapPropagator()
+        context = propagator.extract(headers)
+        # Context is automatically set as current
+
+    def shutdown(self) -> None:
+        """Shutdown OpenTelemetry and flush spans."""
+        if self._tracer_provider:
+            self._tracer_provider.shutdown()
+            logger.info("OpenTelemetry shut down")
+
+
+# Global instance
+_global_otel: Optional[OpenTelemetryManager] = None
+
+
+def get_global_otel() -> Optional[OpenTelemetryManager]:
+    """Get global OpenTelemetry manager."""
+    return _global_otel
+
+
+def initialize_otel(
+    service_name: str = "dory-processor",
+    **kwargs
+) -> OpenTelemetryManager:
+    """
+    Initialize global OpenTelemetry manager.
+
+    Args:
+        service_name: Service name
+        **kwargs: Additional arguments for OpenTelemetryManager
+
+    Returns:
+        Initialized OpenTelemetryManager
+    """
+    global _global_otel
+
+    if _global_otel:
+        logger.warning("Global OpenTelemetry already initialized")
+        return _global_otel
+
+    _global_otel = OpenTelemetryManager(service_name=service_name, **kwargs)
+    _global_otel.initialize()
+
+    logger.info(f"Global OpenTelemetry initialized for service: {service_name}")
+    return _global_otel
+
+
+# Convenience functions using global instance
+
+def get_tracer() -> Optional[trace.Tracer]:
+    """Get tracer from global instance."""
+    otel = get_global_otel()
+    return otel.get_tracer() if otel else None
+
+
+def create_span(
+    name: str,
+    attributes: Optional[Dict[str, Any]] = None,
+    kind: Optional[SpanKind] = None,
+):
+    """Create span using global instance."""
+    otel = get_global_otel()
+    if otel:
+        # Default to INTERNAL kind when OpenTelemetry is available
+        if kind is None and OTEL_AVAILABLE:
+            kind = SpanKind.INTERNAL
+        return otel.create_span(name, attributes, kind)
+    else:
+        # Return no-op context manager
+        @contextmanager
+        def noop():
+            yield None
+        return noop()
+
+
+def trace_function(
+    name: Optional[str] = None,
+    attributes: Optional[Dict[str, Any]] = None,
+    kind: Optional[SpanKind] = None,
+):
+    """Trace function using global instance."""
+    otel = get_global_otel()
+    if otel:
+        # Default to INTERNAL kind when OpenTelemetry is available
+        if kind is None and OTEL_AVAILABLE:
+            kind = SpanKind.INTERNAL
+        return otel.trace(name, attributes, kind)
+    else:
+        # Return no-op decorator
+        def decorator(func):
+            return func
+        return decorator
+
+
+def add_span_attributes(attributes: Dict[str, Any]) -> None:
+    """Add attributes to current span using global instance."""
+    otel = get_global_otel()
+    if otel:
+        otel.add_span_attributes(attributes)
+
+
+def record_exception(exception: Exception) -> None:
+    """Record exception in current span using global instance."""
+    otel = get_global_otel()
+    if otel:
+        otel.record_exception(exception)
+
+
+# Integration with request IDs
+
+def integrate_with_request_id():
+    """
+    Integrate OpenTelemetry with request ID middleware.
+
+    Automatically adds request IDs as span attributes.
+    """
+    try:
+        from dory.middleware.request_id import get_current_request_id
+
+        otel = get_global_otel()
+        if not otel:
+            logger.warning("OpenTelemetry not initialized")
+            return
+
+        # Monkey patch create_span to add request ID
+        original_create_span = otel.create_span
+
+        @contextmanager
+        def create_span_with_request_id(name, attributes=None, kind=None):
+            # Get current request ID
+            request_id = get_current_request_id()
+
+            # Default to INTERNAL kind
+            if kind is None:
+                kind = SpanKind.INTERNAL
+
+            # Add to attributes
+            attrs = attributes or {}
+            if request_id:
+                attrs["request.id"] = request_id
+
+            with original_create_span(name, attrs, kind) as span:
+                yield span
+
+        otel.create_span = create_span_with_request_id
+
+        logger.info("OpenTelemetry integrated with request ID middleware")
+
+    except ImportError:
+        logger.warning("Request ID middleware not available for integration")

dory/output/__init__.py

@@ -0,0 +1,31 @@
+"""Output formatting and publishing for processor results."""
+
+from dory.output.formatter import OutputFormatter, JSONFormatter
+from dory.output.routing import build_routing_key, get_geohash
+from dory.output.envelope import (
+    MessageEnvelope,
+    EnvelopeFormatter,
+    ENVELOPE_SCHEMA_VERSION,
+)
+from dory.output.validator import (
+    EnvelopeValidator,
+    UnsupportedVersionError,
+)
+
+# RabbitMQ requires optional dependency
+try:
+    from dory.output.rabbitmq import RabbitMQPublisher, PublisherConfig, PublishError
+except ImportError:
+    pass
+
+__all__ = [
+    "OutputFormatter",
+    "JSONFormatter",
+    "MessageEnvelope",
+    "EnvelopeFormatter",
+    "EnvelopeValidator",
+    "UnsupportedVersionError",
+    "ENVELOPE_SCHEMA_VERSION",
+    "build_routing_key",
+    "get_geohash",
+]

dory/output/envelope.py

@@ -0,0 +1,137 @@
+"""
+MessageEnvelope for unified publisher/subscriber contract.
+
+Provides a versioned envelope that wraps payload data with metadata,
+enabling schema evolution between publisher and subscriber SDKs.
+
+The envelope contains:
+- schema_version: MAJOR.MINOR version string
+- message_id: Unique UUID for each message
+- timestamp: ISO 8601 creation timestamp
+- payload: The actual business data
+
+Schema versioning rules:
+- MINOR bump (0.1 -> 0.2): Additive change (new optional fields only)
+- MAJOR bump (0.1 -> 1.0): Breaking change (fields removed, types changed)
+- Subscribers should use major version matching for backward compatibility
+"""
+
+import logging
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+logger = logging.getLogger(__name__)
+
+# Current envelope schema version
+ENVELOPE_SCHEMA_VERSION = "0.1"
+
+
+# =========================================================================
+# MessageEnvelope
+# =========================================================================
+
+@dataclass
+class MessageEnvelope:
+    """Versioned envelope wrapping payload data.
+
+    Attributes:
+        schema_version: Envelope schema version (MAJOR.MINOR).
+        message_id: Unique message identifier.
+        timestamp: ISO 8601 timestamp of message creation.
+        payload: The actual message data.
+    """
+
+    schema_version: str = ENVELOPE_SCHEMA_VERSION
+    message_id: str = field(default_factory=lambda: str(uuid4()))
+    timestamp: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+    payload: Any = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert envelope to a plain dictionary."""
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "MessageEnvelope":
+        """Reconstruct an envelope from a dictionary.
+
+        Args:
+            data: Dictionary with envelope fields.
+
+        Returns:
+            MessageEnvelope instance.
+        """
+        return cls(
+            schema_version=data.get("schema_version", ENVELOPE_SCHEMA_VERSION),
+            message_id=data.get("message_id", str(uuid4())),
+            timestamp=data.get("timestamp", ""),
+            payload=data.get("payload", {}),
+        )
+
+
+# =========================================================================
+# EnvelopeFormatter
+# =========================================================================
+
+class EnvelopeFormatter:
+    """Wraps payload data in a MessageEnvelope before formatting.
+
+    Composes with an existing OutputFormatter: the EnvelopeFormatter builds
+    the envelope dict, then delegates serialization to the inner formatter.
+
+    Args:
+        formatter: Inner OutputFormatter for serialization (defaults to JSONFormatter).
+        schema_version: Envelope schema version to use.
+    """
+
+    def __init__(
+        self,
+        formatter: Any = None,
+        schema_version: str = ENVELOPE_SCHEMA_VERSION,
+        **kwargs: Any,
+    ):
+        from dory.output.formatter import JSONFormatter, OutputFormatter
+
+        self._formatter: OutputFormatter = formatter or JSONFormatter()
+        self._schema_version = schema_version
+
+    @property
+    def content_type(self) -> str:
+        return self._formatter.content_type
+
+    def wrap(
+        self,
+        data: Any,
+        **kwargs: Any,
+    ) -> MessageEnvelope:
+        """Wrap data in a MessageEnvelope.
+
+        Args:
+            data: Payload data.
+
+        Returns:
+            MessageEnvelope with payload.
+        """
+        return MessageEnvelope(
+            schema_version=self._schema_version,
+            payload=data,
+        )
+
+    def format(
+        self,
+        data: Any,
+        **kwargs: Any,
+    ) -> bytes:
+        """Wrap data in an envelope and serialize to bytes.
+
+        Args:
+            data: Payload data.
+
+        Returns:
+            Serialized envelope as bytes.
+        """
+        envelope = self.wrap(data)
+        return self._formatter.format(envelope.to_dict())