lucidicai 1.2.15__py3-none-any.whl → 1.2.17__py3-none-any.whl

lucidicai/providers/otel_init.py

@@ -0,0 +1,197 @@
+"""OpenTelemetry initialization and configuration for Lucidic"""
+import logging
+from typing import List, Optional
+
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.instrumentation.openai import OpenAIInstrumentor
+from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor
+from opentelemetry.instrumentation.langchain import LangchainInstrumentor
+
+from .lucidic_span_processor import LucidicSpanProcessor
+from .otel_provider import OpenTelemetryProvider
+from lucidicai.client import Client
+
+logger = logging.getLogger("Lucidic")
+
+
+class LucidicTelemetry:
+    """Manages OpenTelemetry initialization for Lucidic"""
+    
+    _instance = None
+    _initialized = False
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if not self._initialized:
+            self.tracer_provider = None
+            self.span_processor = None
+            self.instrumentors = {}
+            self.provider = OpenTelemetryProvider()
+            self._initialized = True
+    
+    def initialize(self, agent_id: str, service_name: str = "lucidic-ai") -> None:
+        """Initialize OpenTelemetry with Lucidic configuration"""
+        if self.tracer_provider:
+            logger.debug("OpenTelemetry already initialized")
+            return
+            
+        try:
+            # Create resource
+            resource = Resource.create({
+                "service.name": service_name,
+                "service.version": "1.0.0",
+                "lucidic.agent_id": agent_id,
+            })
+            
+            # Create tracer provider
+            self.tracer_provider = TracerProvider(resource=resource)
+            
+            # Add our custom span processor for real-time event handling
+            self.span_processor = LucidicSpanProcessor()
+            self.tracer_provider.add_span_processor(self.span_processor)
+            
+            # Set as global provider
+            trace.set_tracer_provider(self.tracer_provider)
+            
+            logger.info("[LucidicTelemetry] OpenTelemetry initialized")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize OpenTelemetry: {e}")
+            raise
+    
+    def instrument_providers(self, providers: List[str]) -> None:
+        """Instrument specified providers"""
+        for provider in providers:
+            try:
+                if provider == "openai" and provider not in self.instrumentors:
+                    self._instrument_openai()
+                elif provider == "anthropic" and provider not in self.instrumentors:
+                    self._instrument_anthropic()
+                elif provider == "langchain" and provider not in self.instrumentors:
+                    self._instrument_langchain()
+                elif provider == "pydantic_ai":
+                    # Custom instrumentation needed
+                    logger.info(f"[LucidicTelemetry] Pydantic AI will use manual instrumentation")
+                elif provider == "openai_agents":
+                    # OpenAI Agents uses the same OpenAI instrumentation
+                    self._instrument_openai_agents()
+            except Exception as e:
+                logger.error(f"Failed to instrument {provider}: {e}")
+    
+    def _instrument_openai(self) -> None:
+        """Instrument OpenAI"""
+        try:
+            # Get client for masking function
+            client = Client()
+            
+            # Configure instrumentation
+            instrumentor = OpenAIInstrumentor()
+            
+            # Create a custom callback for getting attributes
+            def get_custom_attributes():
+                attrs = {}
+                
+                # Add step context if available
+                if client.session and client.session.active_step:
+                    attrs["lucidic.step_id"] = client.session.active_step.step_id
+                    
+                return attrs
+            
+            instrumentor.instrument(
+                tracer_provider=self.tracer_provider,
+                enrich_token_usage=True,
+                exception_logger=lambda e: logger.error(f"OpenAI error: {e}"),
+                get_common_metrics_attributes=get_custom_attributes,
+                enable_trace_context_propagation=True,
+                use_legacy_attributes=True  # Force legacy attributes mode for now
+            )
+            
+            self.instrumentors["openai"] = instrumentor
+            logger.info("[LucidicTelemetry] Instrumented OpenAI")
+            
+        except Exception as e:
+            logger.error(f"Failed to instrument OpenAI: {e}")
+            raise
+    
+    def _instrument_anthropic(self) -> None:
+        """Instrument Anthropic"""
+        try:
+            instrumentor = AnthropicInstrumentor()
+            
+            # Get client for context
+            client = Client()
+            
+            def get_custom_attributes():
+                attrs = {}
+                if client.session and client.session.active_step:
+                    attrs["lucidic.step_id"] = client.session.active_step.step_id
+                return attrs
+            
+            instrumentor.instrument(
+                tracer_provider=self.tracer_provider,
+                exception_logger=lambda e: logger.error(f"Anthropic error: {e}"),
+                get_common_metrics_attributes=get_custom_attributes,
+                use_legacy_attributes=True  # Force legacy attributes mode
+            )
+            
+            self.instrumentors["anthropic"] = instrumentor
+            logger.info("[LucidicTelemetry] Instrumented Anthropic")
+            
+        except Exception as e:
+            logger.error(f"Failed to instrument Anthropic: {e}")
+            raise
+    
+    def _instrument_langchain(self) -> None:
+        """Instrument LangChain"""
+        try:
+            instrumentor = LangchainInstrumentor()
+            instrumentor.instrument(tracer_provider=self.tracer_provider)
+            
+            self.instrumentors["langchain"] = instrumentor
+            logger.info("[LucidicTelemetry] Instrumented LangChain")
+            
+        except Exception as e:
+            logger.error(f"Failed to instrument LangChain: {e}")
+            raise
+
+    def _instrument_openai_agents(self) -> None:
+        """Instrument OpenAI Agents SDK"""
+        try:
+            from .openai_agents_instrumentor import OpenAIAgentsInstrumentor
+            
+            instrumentor = OpenAIAgentsInstrumentor(tracer_provider=self.tracer_provider)
+            instrumentor.instrument()
+            
+            self.instrumentors["openai_agents"] = instrumentor
+            logger.info("[LucidicTelemetry] Instrumented OpenAI Agents SDK")
+            
+        except Exception as e:
+            logger.error(f"Failed to instrument OpenAI Agents SDK: {e}")
+            raise
+    
+    def uninstrument_all(self) -> None:
+        """Uninstrument all providers"""
+        for name, instrumentor in self.instrumentors.items():
+            try:
+                instrumentor.uninstrument()
+                logger.info(f"[LucidicTelemetry] Uninstrumented {name}")
+            except Exception as e:
+                logger.error(f"Failed to uninstrument {name}: {e}")
+                
+        self.instrumentors.clear()
+        
+        # Shutdown tracer provider
+        if self.tracer_provider:
+            self.tracer_provider.shutdown()
+            self.tracer_provider = None
+            self.span_processor = None
+            
+    def is_initialized(self) -> bool:
+        """Check if telemetry is initialized"""
+        return self.tracer_provider is not None

lucidicai/providers/otel_provider.py

@@ -0,0 +1,168 @@
+"""OpenTelemetry-based provider implementation"""
+import logging
+from typing import Optional, List, Dict, Any
+from contextlib import contextmanager
+
+from opentelemetry import trace, context
+from opentelemetry.trace import Tracer, Span
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.instrumentation.openai import OpenAIInstrumentor
+from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor
+from opentelemetry.instrumentation.langchain import LangchainInstrumentor
+from opentelemetry.semconv_ai import SpanAttributes
+
+from .lucidic_exporter import LucidicSpanExporter
+from .base_providers import BaseProvider
+from lucidicai.client import Client
+
+logger = logging.getLogger("Lucidic")
+
+
+class OpenTelemetryProvider(BaseProvider):
+    """Provider that uses OpenTelemetry instrumentations instead of monkey-patching"""
+    
+    def __init__(self):
+        super().__init__()
+        self._provider_name = "OpenTelemetry"
+        self.tracer_provider = None
+        self.tracer = None
+        self.instrumentors = {}
+        self._active_spans = {}
+        
+    def initialize_telemetry(self, service_name: str = "lucidic-ai", agent_id: str = None) -> None:
+        """Initialize OpenTelemetry with Lucidic exporter"""
+        # Create resource with service info
+        resource = Resource.create({
+            "service.name": service_name,
+            "service.version": "1.0.0",
+            "lucidic.agent_id": agent_id or ""
+        })
+        
+        # Create tracer provider
+        self.tracer_provider = TracerProvider(resource=resource)
+        
+        # Add our custom exporter
+        lucidic_exporter = LucidicSpanExporter()
+        span_processor = BatchSpanProcessor(lucidic_exporter)
+        self.tracer_provider.add_span_processor(span_processor)
+        
+        # Set as global provider
+        trace.set_tracer_provider(self.tracer_provider)
+        
+        # Get tracer
+        self.tracer = trace.get_tracer(__name__)
+        
+    def handle_response(self, response, kwargs, session: Optional = None):
+        """Handle responses - not needed with OTEL approach"""
+        return response
+        
+    def override(self):
+        """Initialize OpenTelemetry instrumentations"""
+        try:
+            client = Client()
+            
+            # Initialize telemetry if not already done
+            if not self.tracer_provider:
+                self.initialize_telemetry(agent_id=client.agent_id)
+            
+            # No actual override needed - instrumentations will be enabled separately
+            logger.info("[OpenTelemetry Provider] Initialized")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize OpenTelemetry: {e}")
+            raise
+            
+    def undo_override(self):
+        """Uninstrument all providers"""
+        for name, instrumentor in self.instrumentors.items():
+            try:
+                instrumentor.uninstrument()
+                logger.info(f"[OpenTelemetry Provider] Uninstrumented {name}")
+            except Exception as e:
+                logger.error(f"Failed to uninstrument {name}: {e}")
+                
+        self.instrumentors.clear()
+        
+        # Shutdown tracer provider
+        if self.tracer_provider:
+            self.tracer_provider.shutdown()
+            
+    def instrument_openai(self) -> None:
+        """Instrument OpenAI with OpenLLMetry"""
+        if "openai" not in self.instrumentors:
+            try:
+                instrumentor = OpenAIInstrumentor()
+                instrumentor.instrument(
+                    tracer_provider=self.tracer_provider,
+                    enrich_token_usage=True,
+                    exception_logger=lambda e: logger.error(f"OpenAI error: {e}")
+                )
+                self.instrumentors["openai"] = instrumentor
+                logger.info("[OpenTelemetry Provider] Instrumented OpenAI")
+            except Exception as e:
+                logger.error(f"Failed to instrument OpenAI: {e}")
+                
+    def instrument_anthropic(self) -> None:
+        """Instrument Anthropic with OpenLLMetry"""
+        if "anthropic" not in self.instrumentors:
+            try:
+                instrumentor = AnthropicInstrumentor()
+                instrumentor.instrument(
+                    tracer_provider=self.tracer_provider,
+                    exception_logger=lambda e: logger.error(f"Anthropic error: {e}")
+                )
+                self.instrumentors["anthropic"] = instrumentor
+                logger.info("[OpenTelemetry Provider] Instrumented Anthropic")
+            except Exception as e:
+                logger.error(f"Failed to instrument Anthropic: {e}")
+                
+    def instrument_langchain(self) -> None:
+        """Instrument LangChain with OpenLLMetry"""
+        if "langchain" not in self.instrumentors:
+            try:
+                instrumentor = LangchainInstrumentor()
+                instrumentor.instrument(tracer_provider=self.tracer_provider)
+                self.instrumentors["langchain"] = instrumentor
+                logger.info("[OpenTelemetry Provider] Instrumented LangChain")
+            except Exception as e:
+                logger.error(f"Failed to instrument LangChain: {e}")
+                
+    def instrument_pydantic_ai(self) -> None:
+        """Instrument Pydantic AI"""
+        # Note: OpenLLMetry doesn't have a Pydantic AI instrumentation yet
+        # We'll need to create custom instrumentation or use manual spans
+        logger.info("[OpenTelemetry Provider] Pydantic AI instrumentation not yet available in OpenLLMetry")
+        
+    @contextmanager
+    def trace_step(self, step_id: str, state: str = None, action: str = None, goal: str = None):
+        """Context manager to associate spans with a specific step"""
+        span = self.tracer.start_span(
+            name=f"step.{step_id}",
+            attributes={
+                "lucidic.step_id": step_id,
+                "lucidic.step.state": state or "",
+                "lucidic.step.action": action or "",
+                "lucidic.step.goal": goal or ""
+            }
+        )
+        
+        token = context.attach(trace.set_span_in_context(span))
+        try:
+            yield span
+        finally:
+            context.detach(token)
+            span.end()
+            
+    def add_image_to_span(self, image_data: str, image_type: str = "screenshot") -> None:
+        """Add image data to current span"""
+        current_span = trace.get_current_span()
+        if current_span and current_span.is_recording():
+            current_span.set_attribute(f"lucidic.image.{image_type}", image_data)
+            
+    def set_step_context(self, step_id: str) -> None:
+        """Set step ID in current span context"""
+        current_span = trace.get_current_span()
+        if current_span and current_span.is_recording():
+            current_span.set_attribute("lucidic.step_id", step_id)

lucidicai/providers/pydantic_ai_handler.py

@@ -1,7 +1,7 @@
 """Pydantic AI provider handler for the Lucidic API"""
 from typing import Any, Dict, Optional
 
-from .base_providers import BaseProvider
+from .legacy.base_providers import BaseProvider
 from lucidicai.client import Client
 from lucidicai.model_pricing import calculate_cost
 from lucidicai.singleton import singleton
@@ -381,11 +381,6 @@ class PydanticAIHandler(BaseProvider):
 
     def _wrap_request(self, model_instance, messages, model_settings, model_request_parameters, original_method):
         """Wrap regular request method to track LLM calls"""
-        # Create event before API call
-        step = Client().session.active_step
-        if step is None:
-            return original_method(model_instance, messages, model_settings, model_request_parameters)
-        
         description = self._format_messages(messages)
         event_id = Client().session.create_event(
             description=description,
@@ -412,13 +407,6 @@ class PydanticAIHandler(BaseProvider):
     
     def _wrap_request_stream_context_manager(self, model_instance, messages, model_settings, model_request_parameters, original_method):
         """Return an async context manager for streaming requests"""
-        # Create event before API call
-        event_id = None
-        step = Client().session.active_step
-        
-        if step is None:
-            return original_method(model_instance, messages, model_settings, model_request_parameters)
-        
         description = self._format_messages(messages)
         event_id = Client().session.create_event(
             description=description,
@@ -466,13 +454,8 @@ class PydanticAIHandler(BaseProvider):
 
     async def _wrap_request_stream(self, model_instance, messages, model_settings, model_request_parameters, original_method):
         """Wrap streaming request method"""
-        # Create event before API call
-        step = Client().session.active_step
-        if step is None:
-            return original_method(model_instance, messages, model_settings, model_request_parameters)
-
         description = self._format_messages(messages)
-        event = step.create_event(
+        event = Client().session.create_event(
             description=description,
             result="Streaming response..."
         )

lucidicai/providers/text_storage.py

@@ -0,0 +1,53 @@
+"""Thread-local storage for text content from multimodal messages"""
+import threading
+from typing import List, Optional
+import logging
+
+logger = logging.getLogger("Lucidic")
+
+# Thread-local storage for text content
+_text_storage = threading.local()
+
+def store_text(text: str, message_index: int = 0) -> None:
+    """Store text content for a message
+    
+    Args:
+        text: The text content to store
+        message_index: The index of the message (default 0)
+    """
+    if not hasattr(_text_storage, 'texts'):
+        _text_storage.texts = {}
+    
+    _text_storage.texts[message_index] = text
+    logger.debug(f"[TextStorage] Stored text for message {message_index}: {text[:50]}...")
+
+def get_stored_text(message_index: int = 0) -> Optional[str]:
+    """Get stored text for a message
+    
+    Args:
+        message_index: The index of the message (default 0)
+        
+    Returns:
+        The stored text or None if not found
+    """
+    if not hasattr(_text_storage, 'texts'):
+        return None
+    
+    return _text_storage.texts.get(message_index)
+
+def get_all_stored_texts() -> dict:
+    """Get all stored texts
+    
+    Returns:
+        Dictionary of message_index -> text
+    """
+    if not hasattr(_text_storage, 'texts'):
+        return {}
+    
+    return _text_storage.texts.copy()
+
+def clear_stored_texts() -> None:
+    """Clear all stored texts"""
+    if hasattr(_text_storage, 'texts'):
+        _text_storage.texts = {}
+        logger.debug("[TextStorage] Cleared all stored texts")