lucidicai 1.2.15__py3-none-any.whl → 1.2.17__py3-none-any.whl

lucidicai/providers/universal_image_interceptor.py

@@ -0,0 +1,276 @@
+"""Universal image interceptor for all LLM providers to capture images from multimodal requests"""
+import logging
+import os
+from typing import Any, Dict, List, Union, Tuple
+from .image_storage import store_image, get_stored_images
+from .text_storage import store_text, clear_stored_texts
+
+logger = logging.getLogger("Lucidic")
+DEBUG = os.getenv("LUCIDIC_DEBUG", "False") == "True"
+
+
+class UniversalImageInterceptor:
+    """Universal image interceptor that can handle different provider formats"""
+    
+    @staticmethod
+    def extract_and_store_images_openai(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Extract and store images from OpenAI-style messages
+        
+        OpenAI format:
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
+            ]
+        }
+        """
+        processed_messages = []
+        clear_stored_texts()  # Clear any previous texts
+        
+        for msg_idx, message in enumerate(messages):
+            if isinstance(message, dict):
+                content = message.get('content')
+                if isinstance(content, list):
+                    # Extract and store text content for multimodal messages
+                    text_parts = []
+                    has_images = False
+                    
+                    for item in content:
+                        if isinstance(item, dict):
+                            if item.get('type') == 'text':
+                                text_parts.append(item.get('text', ''))
+                            elif item.get('type') == 'image_url':
+                                has_images = True
+                                image_data = item.get('image_url', {})
+                                if isinstance(image_data, dict):
+                                    url = image_data.get('url', '')
+                                    if url.startswith('data:image'):
+                                        # Store the image
+                                        placeholder = store_image(url)
+                                        if DEBUG:
+                                            logger.info(f"[Universal Interceptor] Stored OpenAI image, placeholder: {placeholder}")
+                    
+                    # If we have both text and images, store the text separately
+                    if text_parts and has_images:
+                        combined_text = ' '.join(text_parts)
+                        store_text(combined_text, msg_idx)
+                        if DEBUG:
+                            logger.info(f"[Universal Interceptor] Stored text for multimodal message {msg_idx}: {combined_text[:50]}...")
+                
+                processed_messages.append(message)
+        return processed_messages
+    
+    @staticmethod
+    def extract_and_store_images_anthropic(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Extract and store images from Anthropic-style messages
+        
+        Anthropic format:
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "..."}}
+            ]
+        }
+        """
+        processed_messages = []
+        for message in messages:
+            if isinstance(message, dict):
+                content = message.get('content')
+                if isinstance(content, list):
+                    for item in content:
+                        if isinstance(item, dict) and item.get('type') == 'image':
+                            source = item.get('source', {})
+                            if isinstance(source, dict):
+                                data = source.get('data', '')
+                                media_type = source.get('media_type', 'image/jpeg')
+                                if data:
+                                    # Convert to data URL format for consistency
+                                    data_url = f"data:{media_type};base64,{data}"
+                                    placeholder = store_image(data_url)
+                                    if DEBUG:
+                                        logger.info(f"[Universal Interceptor] Stored Anthropic image, placeholder: {placeholder}")
+                processed_messages.append(message)
+        return processed_messages
+    
+    @staticmethod
+    def extract_and_store_images_google(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Extract and store images from Google/Gemini-style messages
+        
+        Google format can vary but often uses:
+        {
+            "parts": [
+                {"text": "What's in this image?"},
+                {"inline_data": {"mime_type": "image/jpeg", "data": "..."}}
+            ]
+        }
+        """
+        processed_messages = []
+        for message in messages:
+            if isinstance(message, dict):
+                parts = message.get('parts', [])
+                for part in parts:
+                    if isinstance(part, dict) and 'inline_data' in part:
+                        inline_data = part['inline_data']
+                        if isinstance(inline_data, dict):
+                            data = inline_data.get('data', '')
+                            mime_type = inline_data.get('mime_type', 'image/jpeg')
+                            if data:
+                                # Convert to data URL format
+                                data_url = f"data:{mime_type};base64,{data}"
+                                placeholder = store_image(data_url)
+                                if DEBUG:
+                                    logger.info(f"[Universal Interceptor] Stored Google image, placeholder: {placeholder}")
+                processed_messages.append(message)
+        return processed_messages
+    
+    @staticmethod
+    def intercept_images(messages: Any, provider: str = "auto") -> Any:
+        """Universal image interception for any provider
+        
+        Args:
+            messages: The messages to process
+            provider: The provider type ("openai", "anthropic", "google", "auto")
+                     If "auto", will try to detect the format
+        
+        Returns:
+            The processed messages (unchanged, but images are stored)
+        """
+        if not messages or not isinstance(messages, list):
+            return messages
+        
+        # Auto-detect provider format if needed
+        if provider == "auto":
+            provider = UniversalImageInterceptor._detect_provider_format(messages)
+        
+        # Process based on provider
+        if provider == "openai":
+            return UniversalImageInterceptor.extract_and_store_images_openai(messages)
+        elif provider == "anthropic":
+            return UniversalImageInterceptor.extract_and_store_images_anthropic(messages)
+        elif provider == "google":
+            return UniversalImageInterceptor.extract_and_store_images_google(messages)
+        else:
+            if DEBUG:
+                logger.warning(f"[Universal Interceptor] Unknown provider format: {provider}")
+            return messages
+    
+    @staticmethod
+    def _detect_provider_format(messages: List[Dict[str, Any]]) -> str:
+        """Detect the provider format based on message structure"""
+        for message in messages:
+            if isinstance(message, dict):
+                # Check for OpenAI format
+                content = message.get('content')
+                if isinstance(content, list):
+                    for item in content:
+                        if isinstance(item, dict):
+                            if item.get('type') == 'image_url' and 'image_url' in item:
+                                return "openai"
+                            elif item.get('type') == 'image' and 'source' in item:
+                                return "anthropic"
+                
+                # Check for Google format
+                if 'parts' in message:
+                    return "google"
+        
+        return "unknown"
+    
+    @staticmethod
+    def create_interceptor(provider: str):
+        """Create a function interceptor for a specific provider
+        
+        Args:
+            provider: The provider type ("openai", "anthropic", "google")
+        
+        Returns:
+            A function that can wrap the provider's API call
+        """
+        def interceptor(original_func):
+            def wrapper(*args, **kwargs):
+                # Extract messages from kwargs (common location)
+                messages = kwargs.get('messages', None)
+                
+                # For Anthropic, messages might be in args[1]
+                if messages is None and len(args) > 1 and isinstance(args[1], list):
+                    messages = args[1]
+                
+                # Intercept images if messages found
+                if messages:
+                    UniversalImageInterceptor.intercept_images(messages, provider)
+                
+                # Call the original function
+                return original_func(*args, **kwargs)
+            
+            return wrapper
+        
+        return interceptor
+    
+    @staticmethod
+    def create_async_interceptor(provider: str):
+        """Create an async function interceptor for a specific provider"""
+        def interceptor(original_func):
+            async def wrapper(*args, **kwargs):
+                # Extract messages from kwargs (common location)
+                messages = kwargs.get('messages', None)
+                
+                # For Anthropic, messages might be in args[1]
+                if messages is None and len(args) > 1 and isinstance(args[1], list):
+                    messages = args[1]
+                
+                # Intercept images if messages found
+                if messages:
+                    UniversalImageInterceptor.intercept_images(messages, provider)
+                
+                # Call the original function
+                return await original_func(*args, **kwargs)
+            
+            return wrapper
+        
+        return interceptor
+
+
+def patch_openai_client(client):
+    """Patch an OpenAI client instance to intercept images"""
+    interceptor = UniversalImageInterceptor.create_interceptor("openai")
+    
+    if hasattr(client, 'chat') and hasattr(client.chat, 'completions'):
+        original_create = client.chat.completions.create
+        client.chat.completions.create = interceptor(original_create)
+        if DEBUG:
+            logger.info("[Universal Interceptor] Patched OpenAI client for image interception")
+    return client
+
+
+def patch_anthropic_client(client):
+    """Patch an Anthropic client instance to intercept images"""
+    interceptor = UniversalImageInterceptor.create_interceptor("anthropic")
+    async_interceptor = UniversalImageInterceptor.create_async_interceptor("anthropic")
+    
+    if hasattr(client, 'messages'):
+        if hasattr(client.messages, 'create'):
+            original_create = client.messages.create
+            client.messages.create = interceptor(original_create)
+        
+        # Handle async version
+        if hasattr(client.messages, 'acreate'):
+            original_acreate = client.messages.acreate
+            client.messages.acreate = async_interceptor(original_acreate)
+        
+        if DEBUG:
+            logger.info("[Universal Interceptor] Patched Anthropic client for image interception")
+    return client
+
+
+def patch_google_client(client):
+    """Patch a Google/Gemini client instance to intercept images"""
+    interceptor = UniversalImageInterceptor.create_interceptor("google")
+    
+    # Google's client structure varies, but often uses generate_content
+    if hasattr(client, 'generate_content'):
+        original_generate = client.generate_content
+        client.generate_content = interceptor(original_generate)
+        if DEBUG:
+            logger.info("[Universal Interceptor] Patched Google client for image interception")
+    return client

lucidicai/session.py

@@ -43,7 +43,8 @@ class Session:
             "task": kwargs.get("task", None),
             "mass_sim_id": kwargs.get("mass_sim_id", None),
             "rubrics": kwargs.get("rubrics", None),
-            "tags": kwargs.get("tags", None)
+            "tags": kwargs.get("tags", None),
+            "production_monitoring": kwargs.get("production_monitoring", False)
         }
         data = Client().make_request('initsession', 'POST', request_data)
         self.session_id = data["session_id"]
@@ -73,11 +74,18 @@ class Session:
             "is_finished": kwargs.get("is_finished", None),
             "task": kwargs.get("task", None),
             "is_successful": kwargs.get("is_successful", None),
-            "is_successful_reason": kwargs.get("is_successful_reason", None),
+            "is_successful_reason": Client().mask(kwargs.get("is_successful_reason", None)),
             "session_eval": kwargs.get("session_eval", None),
-            "session_eval_reason": kwargs.get("session_eval_reason", None),
+            "session_eval_reason": Client().mask(kwargs.get("session_eval_reason", None)),
             "tags": kwargs.get("tags", None)
         }
+
+        # auto end any unfinished steps
+        if kwargs.get("is_finished", None) is True:
+            for step_id, step in self.step_history.items():
+                if not step.is_finished:
+                    self.update_step(step_id=step_id, is_finished=True)
+
         Client().make_request('updatesession', 'PUT', request_data)
 
     def create_step(self, **kwargs) -> str:
@@ -101,12 +109,14 @@ class Session:
 
     def create_event(self, **kwargs):
         # Get step_id from kwargs or active step
+        temp_step_created = False
         if 'step_id' in kwargs and kwargs['step_id'] is not None:
             step_id = kwargs['step_id']
         elif self._active_step:
             step_id = self._active_step
         else:
-            raise InvalidOperationError("No active step to create event in and no step_id provided")
+            step_id = self.create_step()
+            temp_step_created = True
         kwargs.pop('step_id', None)
         event = Event(
             session_id=self.session_id,
@@ -115,6 +125,9 @@ class Session:
         )
         self.event_history[event.event_id] = event
         self._active_event = event
+        if temp_step_created:
+            self.update_step(step_id=step_id, is_finished=True)
+            self._active_step = None
         return event.event_id
 
     def update_event(self, **kwargs):

lucidicai/step.py

@@ -48,11 +48,11 @@ class Step:
             upload_image_to_s3(presigned_url, screenshot, "JPEG")
         request_data = {
             "step_id": self.step_id,
-            "goal": kwargs['goal'] if 'goal' in kwargs else None,
-            "action": kwargs['action'] if 'action' in kwargs else None,
-            "state": kwargs['state'] if 'state' in kwargs else None,
+            "goal": Client().mask(kwargs['goal']) if 'goal' in kwargs else None,
+            "action": Client().mask(kwargs['action']) if 'action' in kwargs else None,
+            "state": Client().mask(kwargs['state']) if 'state' in kwargs else None,
             "eval_score": kwargs['eval_score'] if 'eval_score' in kwargs else None,
-            "eval_description": kwargs['eval_description'] if 'eval_description' in kwargs else None,
+            "eval_description": Client().mask(kwargs['eval_description']) if 'eval_description' in kwargs else None,
             "is_finished": kwargs['is_finished'] if 'is_finished' in kwargs else None,
             "has_screenshot": True if screenshot else None
         }

lucidicai/streaming.py

@@ -14,7 +14,6 @@ class StreamingResponseWrapper:
     
     def __init__(self, response: Any, session: Any, kwargs: Dict[str, Any]):
         self.response = response
-        self.session = session
         self.kwargs = kwargs
         self.chunks = []
         self.start_time = time.time()
@@ -34,7 +33,7 @@ class StreamingResponseWrapper:
                 logger.info(f"[Streaming] Using existing event ID: {self.event_id}")
                 return
                 
-            if self.session and hasattr(self.session, 'active_step') and self.session.active_step:
+            if Client().session:
                 description, images = self._format_messages(self.kwargs.get('messages', ''))
                 
                 event_data = {
@@ -54,7 +53,7 @@ class StreamingResponseWrapper:
                 if images:
                     event_data['screenshots'] = images
                 
-                self.event_id = self.session.create_event(**event_data)
+                self.event_id = Client().session.create_event(**event_data)
                 logger.debug(f"[Streaming] Created new streaming event with ID: {self.event_id}")
         except Exception as e:
             logger.error(f"[Streaming] Error creating initial streaming event: {str(e)}")

lucidicai/telemetry/base_provider.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+class BaseProvider(ABC):
+    def __init__(self):
+        self._provider_name = None
+    
+    @abstractmethod
+    def handle_response(self, response, kwargs, session: Optional = None):
+        """Handle responses from the LLM provider"""
+        pass
+        
+    @abstractmethod
+    def override(self):
+        """Override the provider's API methods"""
+        pass
+        
+    @abstractmethod
+    def undo_override(self):
+        """Restore original API methods"""
+        pass

lucidicai/telemetry/lucidic_exporter.py

@@ -0,0 +1,259 @@
+"""Custom OpenTelemetry exporter for Lucidic backend compatibility"""
+import json
+import logging
+from typing import Sequence, Optional, Dict, Any, List
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+from opentelemetry.trace import StatusCode
+from opentelemetry.semconv_ai import SpanAttributes
+
+from lucidicai.client import Client
+from lucidicai.model_pricing import calculate_cost
+from lucidicai.image_upload import extract_base64_images
+
+logger = logging.getLogger("Lucidic")
+import os
+
+DEBUG = os.getenv("LUCIDIC_DEBUG", "False") == "True"
+VERBOSE = os.getenv("LUCIDIC_VERBOSE", "False") == "True"
+
+
+class LucidicSpanExporter(SpanExporter):
+    """Custom exporter that converts OpenTelemetry spans to Lucidic events"""
+    
+    def __init__(self):
+        self.pending_events = {}  # Track events by span_id
+        
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        """Export spans by converting them to Lucidic events"""
+        try:
+            client = Client()
+            if not client.session:
+                logger.debug("No active session, skipping span export")
+                return SpanExportResult.SUCCESS
+                
+            for span in spans:
+                self._process_span(span, client)
+                
+            return SpanExportResult.SUCCESS
+        except Exception as e:
+            logger.error(f"Failed to export spans: {e}")
+            return SpanExportResult.FAILURE
+    
+    def _process_span(self, span: ReadableSpan, client: Client) -> None:
+        """Process a single span and convert to Lucidic event"""
+        try:
+            # Skip non-LLM spans
+            if not self._is_llm_span(span):
+                return
+                
+            # Extract relevant attributes
+            attributes = dict(span.attributes or {})
+            
+            # Create or update event based on span lifecycle
+            span_id = format(span.context.span_id, '016x')
+            
+            if span_id not in self.pending_events:
+                # New span - create event
+                event_id = self._create_event_from_span(span, attributes, client)
+                if event_id:
+                    self.pending_events[span_id] = {
+                        'event_id': event_id,
+                        'start_time': span.start_time
+                    }
+            else:
+                # Span ended - update event
+                event_info = self.pending_events.pop(span_id)
+                self._update_event_from_span(span, attributes, event_info['event_id'], client)
+                
+        except Exception as e:
+            logger.error(f"Failed to process span {span.name}: {e}")
+    
+    def _is_llm_span(self, span: ReadableSpan) -> bool:
+        """Check if this is an LLM-related span"""
+        # Check span name patterns
+        llm_patterns = ['openai', 'anthropic', 'chat', 'completion', 'embedding', 'llm']
+        span_name_lower = span.name.lower()
+        
+        if any(pattern in span_name_lower for pattern in llm_patterns):
+            return True
+            
+        # Check for LLM attributes
+        if span.attributes:
+            for key in span.attributes:
+                if key.startswith('gen_ai.') or key.startswith('llm.'):
+                    return True
+                    
+        return False
+    
+    def _create_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any], client: Client) -> Optional[str]:
+        """Create a Lucidic event from span start"""
+        try:
+            # Extract description from prompts/messages
+            description = self._extract_description(span, attributes)
+            
+            # Extract images if present
+            images = self._extract_images(attributes)
+            
+            # Get model info
+            model = attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) or \
+                   attributes.get(SpanAttributes.LLM_REQUEST_MODEL) or \
+                   attributes.get('gen_ai.request.model') or 'unknown'
+            
+            # Create event
+            event_kwargs = {
+                'description': description,
+                'result': "Processing...",  # Will be updated when span ends
+                'model': model
+            }
+            
+            if images:
+                event_kwargs['screenshots'] = images
+                
+            # Check if we have a specific step_id in span attributes
+            step_id = attributes.get('lucidic.step_id')
+            if step_id:
+                event_kwargs['step_id'] = step_id
+                
+            return client.session.create_event(**event_kwargs)
+            
+        except Exception as e:
+            logger.error(f"Failed to create event from span: {e}")
+            return None
+    
+    def _update_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any], event_id: str, client: Client) -> None:
+        """Update a Lucidic event from span end"""
+        try:
+            # Extract response/result
+            result = self._extract_result(span, attributes)
+            
+            # Calculate cost if we have token usage
+            cost = self._calculate_cost(attributes)
+            
+            # Determine success
+            is_successful = span.status.status_code != StatusCode.ERROR
+            
+            update_kwargs = {
+                'event_id': event_id,
+                'result': result,
+                'is_finished': True,
+                'is_successful': is_successful
+            }
+            
+            if cost is not None:
+                update_kwargs['cost_added'] = cost
+                
+            client.session.update_event(**update_kwargs)
+            
+        except Exception as e:
+            logger.error(f"Failed to update event from span: {e}")
+    
+    def _extract_description(self, span: ReadableSpan, attributes: Dict[str, Any]) -> str:
+        """Extract description from span attributes"""
+        # Try to get prompts/messages
+        prompts = attributes.get(SpanAttributes.LLM_PROMPTS) or \
+                 attributes.get('gen_ai.prompt')
+        
+        if VERBOSE:
+            logger.info(f"[SpaneExporter -- DEBUG] Extracting Description attributes: {attributes}, prompts: {prompts}")
+
+        if prompts:
+            if isinstance(prompts, list) and prompts:
+                # Handle message list format
+                return self._format_messages(prompts)
+            elif isinstance(prompts, str):
+                return prompts
+                
+        # Fallback to span name
+        return f"LLM Call: {span.name}"
+    
+    def _extract_result(self, span: ReadableSpan, attributes: Dict[str, Any]) -> str:
+        """Extract result/response from span attributes"""
+        # Try to get completions
+        completions = attributes.get(SpanAttributes.LLM_COMPLETIONS) or \
+                     attributes.get('gen_ai.completion')
+        
+        if completions:
+            if isinstance(completions, list) and completions:
+                # Handle multiple completions
+                return "\n".join(str(c) for c in completions)
+            elif isinstance(completions, str):
+                return completions
+                
+        # Check for error
+        if span.status.status_code == StatusCode.ERROR:
+            return f"Error: {span.status.description or 'Unknown error'}"
+            
+        return "Response received"
+    
+    def _extract_images(self, attributes: Dict[str, Any]) -> List[str]:
+        """Extract base64 images from attributes"""
+        images = []
+        
+        # Check prompts for multimodal content
+        prompts = attributes.get(SpanAttributes.LLM_PROMPTS) or \
+                 attributes.get('gen_ai.prompt')
+                 
+        if isinstance(prompts, list):
+            for prompt in prompts:
+                if isinstance(prompt, dict) and 'content' in prompt:
+                    content = prompt['content']
+                    if isinstance(content, list):
+                        for item in content:
+                            if isinstance(item, dict) and item.get('type') == 'image_url':
+                                image_url = item.get('image_url', {})
+                                if isinstance(image_url, dict) and 'url' in image_url:
+                                    url = image_url['url']
+                                    if url.startswith('data:image'):
+                                        images.append(url)
+                                        
+        return images
+    
+    def _format_messages(self, messages: List[Any]) -> str:
+        """Format message list into description"""
+        formatted = []
+        
+        for msg in messages:
+            if isinstance(msg, dict):
+                role = msg.get('role', 'unknown')
+                content = msg.get('content', '')
+                
+                if isinstance(content, str):
+                    formatted.append(f"{role}: {content}")
+                elif isinstance(content, list):
+                    # Handle multimodal content
+                    text_parts = []
+                    for item in content:
+                        if isinstance(item, dict) and item.get('type') == 'text':
+                            text_parts.append(item.get('text', ''))
+                    if text_parts:
+                        formatted.append(f"{role}: {' '.join(text_parts)}")
+                        
+        return '\n'.join(formatted) if formatted else "Model request"
+    
+    def _calculate_cost(self, attributes: Dict[str, Any]) -> Optional[float]:
+        """Calculate cost from token usage"""
+        prompt_tokens = attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) or \
+                       attributes.get('gen_ai.usage.prompt_tokens') or 0
+        completion_tokens = attributes.get(SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) or \
+                           attributes.get('gen_ai.usage.completion_tokens') or 0
+        
+        if prompt_tokens or completion_tokens:
+            model = attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) or \
+                   attributes.get(SpanAttributes.LLM_REQUEST_MODEL) or \
+                   attributes.get('gen_ai.request.model')
+                   
+            if model:
+                return calculate_cost(prompt_tokens, completion_tokens, model)
+                
+        return None
+    
+    def shutdown(self) -> None:
+        """Shutdown the exporter"""
+        # Process any remaining pending events
+        if self.pending_events:
+            logger.warning(f"Shutting down with {len(self.pending_events)} pending events")
+            
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """Force flush any pending spans"""
+        return True