lucidicai 2.0.2__py3-none-any.whl → 2.1.1__py3-none-any.whl

lucidicai/telemetry/context_bridge.py

@@ -0,0 +1,82 @@
+"""Bridge between Lucidic context and OpenTelemetry context.
+
+This module ensures that Lucidic's contextvars (session_id, parent_event_id)
+are properly propagated through OpenTelemetry's context system, which is
+necessary for instrumentors that create spans in different execution contexts.
+"""
+
+from typing import Optional
+from opentelemetry import baggage, context as otel_context
+from opentelemetry.trace import set_span_in_context
+from ..utils.logger import debug, verbose, truncate_id
+
+
+def inject_lucidic_context() -> otel_context.Context:
+    """Inject Lucidic context into OpenTelemetry baggage.
+    
+    This ensures that our context variables are available to any spans
+    created by OpenTelemetry instrumentors, even if they run in different
+    threads or async contexts.
+    
+    Returns:
+        OpenTelemetry Context with Lucidic values in baggage
+    """
+    try:
+        from ..sdk.context import current_session_id, current_parent_event_id
+        
+        ctx = otel_context.get_current()
+        
+        # Get Lucidic context values
+        session_id = None
+        parent_event_id = None
+        
+        try:
+            session_id = current_session_id.get(None)
+        except Exception:
+            pass
+            
+        try:
+            parent_event_id = current_parent_event_id.get(None)
+        except Exception:
+            pass
+        
+        # Inject into OpenTelemetry baggage
+        if session_id:
+            ctx = baggage.set_baggage("lucidic.session_id", session_id, context=ctx)
+            debug(f"[ContextBridge] Injected session_id {truncate_id(session_id)} into OTel baggage")
+            
+        if parent_event_id:
+            ctx = baggage.set_baggage("lucidic.parent_event_id", parent_event_id, context=ctx)
+            debug(f"[ContextBridge] Injected parent_event_id {truncate_id(parent_event_id)} into OTel baggage")
+        
+        return ctx
+        
+    except Exception as e:
+        verbose(f"[ContextBridge] Failed to inject context: {e}")
+        return otel_context.get_current()
+
+
+def extract_lucidic_context(ctx: Optional[otel_context.Context] = None) -> tuple[Optional[str], Optional[str]]:
+    """Extract Lucidic context from OpenTelemetry baggage.
+    
+    Args:
+        ctx: OpenTelemetry context (uses current if not provided)
+        
+    Returns:
+        Tuple of (session_id, parent_event_id)
+    """
+    if ctx is None:
+        ctx = otel_context.get_current()
+    
+    try:
+        session_id = baggage.get_baggage("lucidic.session_id", context=ctx)
+        parent_event_id = baggage.get_baggage("lucidic.parent_event_id", context=ctx)
+        
+        if session_id or parent_event_id:
+            debug(f"[ContextBridge] Extracted from OTel baggage - session: {truncate_id(session_id)}, parent: {truncate_id(parent_event_id)}")
+        
+        return session_id, parent_event_id
+        
+    except Exception as e:
+        verbose(f"[ContextBridge] Failed to extract context: {e}")
+        return None, None

lucidicai/telemetry/context_capture_processor.py

@@ -8,13 +8,11 @@ processed asynchronously in different threads/contexts.
 This fixes the nesting issue for ALL providers (OpenAI, Anthropic, LangChain, etc.)
 """
 
-import logging
 from typing import Optional
 from opentelemetry.sdk.trace import SpanProcessor, ReadableSpan
 from opentelemetry.trace import Span
 from opentelemetry import context as otel_context
-
-logger = logging.getLogger("Lucidic")
+from ..utils.logger import debug, verbose, truncate_id
 
 
 class ContextCaptureProcessor(SpanProcessor):
@@ -24,33 +22,51 @@ class ContextCaptureProcessor(SpanProcessor):
         """Called when a span is started - capture context here."""
         try:
             # Import here to avoid circular imports
-            from lucidicai.context import current_session_id, current_parent_event_id
+            from lucidicai.sdk.context import current_session_id, current_parent_event_id
+            from .context_bridge import extract_lucidic_context
             
-            # Capture session ID from context
+            # Try to get from contextvars first
             session_id = None
+            parent_event_id = None
+            
             try:
                 session_id = current_session_id.get(None)
             except Exception:
                 pass
             
-            # Capture parent event ID from context
-            parent_event_id = None
             try:
                 parent_event_id = current_parent_event_id.get(None)
             except Exception:
                 pass
             
+            # If not found in contextvars, try OpenTelemetry baggage
+            # This handles cases where spans are created in different threads
+            if not session_id or not parent_event_id:
+                baggage_session, baggage_parent = extract_lucidic_context(parent_context)
+                if not session_id and baggage_session:
+                    session_id = baggage_session
+                    debug(f"[ContextCapture] Got session_id from OTel baggage for span {span.name}")
+                if not parent_event_id and baggage_parent:
+                    parent_event_id = baggage_parent
+                    debug(f"[ContextCapture] Got parent_event_id from OTel baggage for span {span.name}")
+            
+            # Add debug logging to understand context propagation
+            debug(f"[ContextCapture] Processing span '{span.name}' - session: {truncate_id(session_id)}, parent: {truncate_id(parent_event_id)}")
+            
             # Store in span attributes for later retrieval
             if session_id:
                 span.set_attribute("lucidic.session_id", session_id)
+                debug(f"[ContextCapture] Set session_id attribute for span {span.name}")
             
             if parent_event_id:
                 span.set_attribute("lucidic.parent_event_id", parent_event_id)
-                logger.debug(f"[ContextCapture] Captured parent_event_id {parent_event_id[:8]}... for span {span.name}")
+                debug(f"[ContextCapture] Captured parent_event_id {truncate_id(parent_event_id)} for span {span.name}")
+            else:
+                debug(f"[ContextCapture] No parent_event_id available for span {span.name}")
             
         except Exception as e:
             # Never fail span creation due to context capture
-            logger.debug(f"[ContextCapture] Failed to capture context: {e}")
+            verbose(f"[ContextCapture] Failed to capture context: {e}")
     
     def on_end(self, span: ReadableSpan) -> None:
         """Called when a span ends - no action needed."""

lucidicai/telemetry/litellm_bridge.py

@@ -14,9 +14,10 @@ except ImportError:
         def __init__(self, **kwargs):
             pass
 
-from lucidicai.client import Client
-from lucidicai.model_pricing import calculate_cost
-from lucidicai.context import current_parent_event_id
+from lucidicai.sdk.event import create_event
+from lucidicai.sdk.init import get_session_id
+from lucidicai.telemetry.utils.model_pricing import calculate_cost
+from lucidicai.sdk.context import current_parent_event_id
 
 logger = logging.getLogger("Lucidic")
 DEBUG = os.getenv("LUCIDIC_DEBUG", "False") == "True"
@@ -76,16 +77,12 @@ class LucidicLiteLLMCallback(CustomLogger):
     def log_pre_api_call(self, model, messages, kwargs):
         """Called before the LLM API call"""
         try:
-            client = Client()
-            if not client.session:
+            session_id = get_session_id()
+            if not session_id:
                 return
                 
             # Extract description from messages
             description = self._format_messages(messages)
-            
-            # Apply masking if configured
-            if hasattr(client, 'mask') and callable(client.mask):
-                description = client.mask(description)
                 
             # Store pre-call info for later use
             call_id = kwargs.get("litellm_call_id", str(time.time())) if kwargs else str(time.time())
@@ -109,8 +106,8 @@ class LucidicLiteLLMCallback(CustomLogger):
         self._register_callback(callback_id)
         
         try:
-            client = Client()
-            if not client.session:
+            session_id = get_session_id()
+            if not session_id:
                 self._complete_callback(callback_id)
                 return
                 
@@ -129,10 +126,6 @@ class LucidicLiteLLMCallback(CustomLogger):
             # Extract response content
             result = self._extract_response_content(response_obj)
             
-            # Apply masking to result if configured
-            if hasattr(client, 'mask') and callable(client.mask):
-                result = client.mask(result)
-            
             # Calculate cost if usage info is available
             usage = self._extract_usage(response_obj)
             cost = None
@@ -142,7 +135,7 @@ class LucidicLiteLLMCallback(CustomLogger):
             # Extract any images from multimodal requests
             images = self._extract_images_from_messages(messages)
             
-            # Create LLM_GENERATION typed event
+            # Get parent event ID from context
             parent_id = None
             try:
                 parent_id = current_parent_event_id.get(None)
@@ -150,11 +143,13 @@ class LucidicLiteLLMCallback(CustomLogger):
                 parent_id = None
 
             # occurred_at/duration from datetimes
-            occ_dt = start_time if isinstance(start_time, datetime) else None
+            occ_dt = start_time.isoformat() if isinstance(start_time, datetime) else None
             duration_secs = (end_time - start_time).total_seconds() if isinstance(start_time, datetime) and isinstance(end_time, datetime) else None
 
-            client.create_event(
+            # Create event with correct field names
+            create_event(
                 type="llm_generation",
+                session_id=session_id,  # Pass session_id explicitly
                 provider=provider,
                 model=model,
                 messages=messages,
@@ -162,7 +157,7 @@ class LucidicLiteLLMCallback(CustomLogger):
                 input_tokens=(usage or {}).get("prompt_tokens", 0),
                 output_tokens=(usage or {}).get("completion_tokens", 0),
                 cost=cost,
-                parent_event_id=parent_id,
+                parent_event_id=parent_id,  # This will be normalized by EventBuilder
                 occurred_at=occ_dt,
                 duration=duration_secs,
             )
@@ -185,8 +180,8 @@ class LucidicLiteLLMCallback(CustomLogger):
         self._register_callback(callback_id)
         
         try:
-            client = Client()
-            if not client.session:
+            session_id = get_session_id()
+            if not session_id:
                 self._complete_callback(callback_id)
                 return
                 
@@ -211,14 +206,15 @@ class LucidicLiteLLMCallback(CustomLogger):
                 parent_id = current_parent_event_id.get(None)
             except Exception:
                 parent_id = None
-            occ_dt = start_time if isinstance(start_time, datetime) else None
+            occ_dt = start_time.isoformat() if isinstance(start_time, datetime) else None
             duration_secs = (end_time - start_time).total_seconds() if isinstance(start_time, datetime) and isinstance(end_time, datetime) else None
 
-            client.create_event(
+            create_event(
                 type="error_traceback",
+                session_id=session_id,  # Pass session_id explicitly
                 error=error_msg,
                 traceback="",
-                parent_event_id=parent_id,
+                parent_event_id=parent_id,  # This will be normalized by EventBuilder
                 occurred_at=occ_dt,
                 duration=duration_secs,
                 metadata={"provider": provider, "litellm": True}

lucidicai/telemetry/lucidic_exporter.py

@@ -4,7 +4,6 @@ Converts completed spans into immutable typed LLM events via Client.create_event
 which enqueues non-blocking delivery through the EventQueue.
 """
 import json
-import logging
 from typing import Sequence, Optional, Dict, Any, List
 from datetime import datetime, timezone
 from opentelemetry.sdk.trace import ReadableSpan
@@ -12,16 +11,12 @@ from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
 from opentelemetry.trace import StatusCode
 from opentelemetry.semconv_ai import SpanAttributes
 
-from lucidicai.client import Client
-from lucidicai.context import current_session_id, current_parent_event_id
-from lucidicai.model_pricing import calculate_cost
+from ..sdk.event import create_event
+from ..sdk.init import get_session_id
+from ..sdk.context import current_session_id, current_parent_event_id
+from ..telemetry.utils.model_pricing import calculate_cost
 from .extract import detect_is_llm_span, extract_images, extract_prompts, extract_completions, extract_model
-
-logger = logging.getLogger("Lucidic")
-import os
-
-DEBUG = os.getenv("LUCIDIC_DEBUG", "False") == "True"
-VERBOSE = os.getenv("LUCIDIC_VERBOSE", "False") == "True"
+from ..utils.logger import debug, info, warning, error, verbose, truncate_id
 
 
 class LucidicSpanExporter(SpanExporter):
@@ -29,26 +24,35 @@ class LucidicSpanExporter(SpanExporter):
 
     def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         try:
-            client = Client()
-            if DEBUG and spans:
-                logger.debug(f"[LucidicSpanExporter] Processing {len(spans)} spans")
+            if spans:
+                debug(f"[Telemetry] Processing {len(spans)} OpenTelemetry spans")
             for span in spans:
-                self._process_span(span, client)
-            if DEBUG and spans:
-                logger.debug(f"[LucidicSpanExporter] Successfully exported {len(spans)} spans")
+                self._process_span(span)
+            if spans:
+                debug(f"[Telemetry] Successfully exported {len(spans)} spans")
             return SpanExportResult.SUCCESS
         except Exception as e:
-            logger.error(f"Failed to export spans: {e}")
+            error(f"[Telemetry] Failed to export spans: {e}")
             return SpanExportResult.FAILURE
 
-    def _process_span(self, span: ReadableSpan, client: Client) -> None:
+    def _process_span(self, span: ReadableSpan) -> None:
         """Convert a single LLM span into a typed, immutable event."""
         try:
             if not detect_is_llm_span(span):
+                verbose(f"[Telemetry] Skipping non-LLM span: {span.name}")
                 return
 
+            debug(f"[Telemetry] Processing LLM span: {span.name}")
+
             attributes = dict(span.attributes or {})
 
+            # Skip spans that are likely duplicates or incomplete
+            # Check if this is a responses.parse span that was already handled
+            if span.name == "openai.responses.create" and not attributes.get("lucidic.instrumented"):
+                # This might be from incorrect standard instrumentation
+                verbose(f"[Telemetry] Skipping potentially duplicate responses span without our marker")
+                return
+
             # Resolve session id
             target_session_id = attributes.get('lucidic.session_id')
             if not target_session_id:
@@ -56,22 +60,30 @@ class LucidicSpanExporter(SpanExporter):
                     target_session_id = current_session_id.get(None)
                 except Exception:
                     target_session_id = None
-            if not target_session_id and getattr(client, 'session', None) and getattr(client.session, 'session_id', None):
-                target_session_id = client.session.session_id
             if not target_session_id:
+                target_session_id = get_session_id()
+            if not target_session_id:
+                debug(f"[Telemetry] No session ID for span {span.name}, skipping")
                 return
 
             # Parent nesting - get from span attributes (captured at span creation)
             parent_id = attributes.get('lucidic.parent_event_id')
+            debug(f"[Telemetry] Span {span.name} has parent_id from attributes: {truncate_id(parent_id)}")
             if not parent_id:
                 # Fallback to trying context (may work if same thread)
                 try:
                     parent_id = current_parent_event_id.get(None)
+                    if parent_id:
+                        debug(f"[Telemetry] Got parent_id from context for span {span.name}: {truncate_id(parent_id)}")
                 except Exception:
                     parent_id = None
+            
+            if not parent_id:
+                debug(f"[Telemetry] No parent_id available for span {span.name}")
 
             # Timing
-            occurred_at = datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=timezone.utc) if span.start_time else datetime.now(tz=timezone.utc)
+            occurred_at_dt = datetime.fromtimestamp(span.start_time / 1_000_000_000, tz=timezone.utc) if span.start_time else datetime.now(tz=timezone.utc)
+            occurred_at = occurred_at_dt.isoformat()  # Convert to ISO string for JSON serialization
             duration_seconds = ((span.end_time - span.start_time) / 1_000_000_000) if (span.start_time and span.end_time) else None
 
             # Typed fields using extract utilities
@@ -79,17 +91,36 @@ class LucidicSpanExporter(SpanExporter):
             provider = self._detect_provider_name(attributes)
             messages = extract_prompts(attributes) or []
             params = self._extract_params(attributes)
-            output_text = extract_completions(span, attributes) or "Response received"
+            output_text = extract_completions(span, attributes)
+
+            # Skip spans with no meaningful output (likely incomplete or duplicate instrumentation)
+            if not output_text or output_text == "Response received":
+                # Only use "Response received" if we have other meaningful data
+                if not messages and not attributes.get("lucidic.instrumented"):
+                    verbose(f"[Telemetry] Skipping span {span.name} with no meaningful content")
+                    return
+                # Use a more descriptive default if we must
+                if not output_text:
+                    output_text = "Response received"
+
             input_tokens = self._extract_prompt_tokens(attributes)
             output_tokens = self._extract_completion_tokens(attributes)
             cost = self._calculate_cost(attributes)
             images = extract_images(attributes)
 
-            # Create immutable event via non-blocking queue
-            event_id = client.create_event(
+            # Set context for parent if needed
+            from ..sdk.context import current_parent_event_id as parent_context
+            if parent_id:
+                token = parent_context.set(parent_id)
+            else:
+                token = None
+            
+            try:
+                # Create immutable event via non-blocking queue
+                debug(f"[Telemetry] Creating LLM event with parent_id: {truncate_id(parent_id)}, session_id: {truncate_id(target_session_id)}")
+                event_id = create_event(
                 type="llm_generation",
-                session_id=target_session_id,
-                parent_event_id=parent_id,
+                session_id=target_session_id,  # Pass the session_id explicitly
                 occurred_at=occurred_at,
                 duration=duration_seconds,
                 provider=provider,
@@ -101,16 +132,20 @@ class LucidicSpanExporter(SpanExporter):
                 output_tokens=output_tokens,
                 cost=cost,
                 raw={"images": images} if images else None,
+                parent_event_id=parent_id,  # Pass the parent_id explicitly
             )
+            finally:
+                # Reset parent context
+                if token:
+                    parent_context.reset(token)
             
-            if DEBUG:
-                logger.debug(f"[LucidicSpanExporter] Created LLM event {event_id} for session {target_session_id[:8]}...")
+            debug(f"[Telemetry] Created LLM event {truncate_id(event_id)} from span {span.name} for session {truncate_id(target_session_id)}")
 
         except Exception as e:
-            logger.error(f"Failed to process span {span.name}: {e}")
+            error(f"[Telemetry] Failed to process span {span.name}: {e}")
     
     
-    def _create_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any], client: Client) -> Optional[str]:
+    def _create_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any]) -> Optional[str]:
         """Create a Lucidic event from span start"""
         try:
             # Extract description from prompts/messages
@@ -132,28 +167,29 @@ class LucidicSpanExporter(SpanExporter):
                 except Exception:
                     target_session_id = None
             if not target_session_id:
-                if getattr(client, 'session', None) and getattr(client.session, 'session_id', None):
-                    target_session_id = client.session.session_id
+                target_session_id = get_session_id()
             if not target_session_id:
+                debug(f"[Telemetry] No session ID for span {span.name}, skipping")
                 return None
 
             # Create event
             event_kwargs = {
+                'session_id': target_session_id,  # Pass session_id explicitly
                 'description': description,
                 'result': "Processing...",  # Will be updated when span ends
                 'model': model
             }
-            
+
             if images:
                 event_kwargs['screenshots'] = images
-                
-            return client.create_event_for_session(target_session_id, **event_kwargs)
+
+            return create_event(**event_kwargs)
             
         except Exception as e:
-            logger.error(f"Failed to create event from span: {e}")
+            error(f"[Telemetry] Failed to create event from span: {e}")
             return None
     
-    def _update_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any], event_id: str, client: Client) -> None:
+    def _update_event_from_span(self, span: ReadableSpan, attributes: Dict[str, Any], event_id: str) -> None:
         """Deprecated: events are immutable; no updates performed."""
         return
     
@@ -163,8 +199,7 @@ class LucidicSpanExporter(SpanExporter):
         prompts = attributes.get(SpanAttributes.LLM_PROMPTS) or \
                  attributes.get('gen_ai.prompt')
         
-        if VERBOSE:
-            logger.info(f"[SpaneExporter -- DEBUG] Extracting Description attributes: {attributes}, prompts: {prompts}")
+        verbose(f"[Telemetry] Extracting description from attributes: {attributes}, prompts: {prompts}")
 
         if prompts:
             if isinstance(prompts, list) and prompts:
@@ -210,31 +245,35 @@ class LucidicSpanExporter(SpanExporter):
         }
 
     def _extract_prompt_tokens(self, attributes: Dict[str, Any]) -> int:
-        return (
-            attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) or
-            attributes.get('gen_ai.usage.prompt_tokens') or
-            attributes.get('gen_ai.usage.input_tokens') or 0
-        )
+        # Check each attribute and return the first non-None value
+        value = attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS)
+        if value is not None:
+            return value
+        value = attributes.get('gen_ai.usage.prompt_tokens')
+        if value is not None:
+            return value
+        value = attributes.get('gen_ai.usage.input_tokens')
+        if value is not None:
+            return value
+        return 0
 
     def _extract_completion_tokens(self, attributes: Dict[str, Any]) -> int:
-        return (
-            attributes.get(SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) or
-            attributes.get('gen_ai.usage.completion_tokens') or
-            attributes.get('gen_ai.usage.output_tokens') or 0
-        )
+        # Check each attribute and return the first non-None value
+        value = attributes.get(SpanAttributes.LLM_USAGE_COMPLETION_TOKENS)
+        if value is not None:
+            return value
+        value = attributes.get('gen_ai.usage.completion_tokens')
+        if value is not None:
+            return value
+        value = attributes.get('gen_ai.usage.output_tokens')
+        if value is not None:
+            return value
+        return 0
     
     def _calculate_cost(self, attributes: Dict[str, Any]) -> Optional[float]:
-        prompt_tokens = (
-            attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) or
-            attributes.get('gen_ai.usage.prompt_tokens') or
-            attributes.get('gen_ai.usage.input_tokens') or 0
-        )
-        completion_tokens = (
-            attributes.get(SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) or
-            attributes.get('gen_ai.usage.completion_tokens') or
-            attributes.get('gen_ai.usage.output_tokens') or 0
-        )
-        total_tokens = (prompt_tokens or 0) + (completion_tokens or 0)
+        prompt_tokens = self._extract_prompt_tokens(attributes)
+        completion_tokens = self._extract_completion_tokens(attributes)
+        total_tokens = prompt_tokens + completion_tokens
         if total_tokens > 0:
             model = (
                 attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) or