synth-ai 0.2.3__py3-none-any.whl → 0.2.4.dev2__py3-none-any.whl

synth_ai/tracing_v3/abstractions.py

@@ -18,13 +18,27 @@ Session Structure:
 - SessionTrace: Top-level container for a complete session
   - SessionTimeStep: Logical steps within a session (e.g., conversation turns)
     - Events: Individual events that occurred during the timestep
-    - Messages: User/assistant messages exchanged
+    - Messages: Information passed between subsystems (user, agent, runtime, environments)
+
+Concepts:
+---------
+- Events capture something that happened inside a subsystem. They may or may not be externally
+  visible. Examples include an LLM API call (LMCAISEvent), a tool selection (RuntimeEvent), or
+  a tool execution outcome (EnvironmentEvent).
+
+- Messages represent information transmitted between subsystems within the session.
+  Messages are used to record communications like: a user sending input to the agent,
+  the agent/runtime sending a tool invocation to an environment, the environment sending a
+  tool result back, and the agent sending a reply to the user. Do not confuse these with
+  provider-specific LLM API "messages" (prompt formatting) — those belong inside an LMCAISEvent
+  as part of its input/output content, not as SessionEventMessages.
 """
 
 from __future__ import annotations
 from dataclasses import dataclass, field, asdict
 from datetime import datetime
 from typing import Any, Dict, List, Optional
+from .lm_call_record_abstractions import LLMCallRecord
 
 
 @dataclass
@@ -46,18 +60,39 @@ class TimeRecord:
 
 
 @dataclass
-class SessionEventMessage:
-    """Message exchanged during a session.
+class SessionEventMarkovBlanketMessage:
+    """Message crossing Markov blanket boundaries between systems in a session.
+    
+    IMPORTANT: This represents information transfer BETWEEN distinct systems/subsystems,
+    where each system is conceptualized as having a Markov blanket that separates its
+    internal states from the external environment. These messages cross those boundaries.
+    
+    This is NOT for chat messages within an LLM conversation (those belong in LLMCallRecord).
+    Instead, this captures inter-system communication such as:
+    - Human -> Agent system (user providing instructions)
+    - Agent -> Runtime (agent deciding on an action)
+    - Runtime -> Environment (executing a tool/action)
+    - Environment -> Runtime (returning results)
+    - Runtime -> Agent (passing back results)
+    - Agent -> Human (final response)
     
-    Represents any message passed between participants in a session, including
-    user inputs, assistant responses, and system messages.
+    Each system maintains its own internal state and processing, but can only influence
+    other systems through these explicit boundary-crossing messages. This follows the
+    Free Energy Principle where systems minimize surprise by maintaining boundaries.
     
     Attributes:
-        content: The actual message content (text, JSON, etc.)
-        message_type: Type identifier (e.g., 'user', 'assistant', 'system', 'tool')
-        time_record: Timing information for the message
-        metadata: Additional message metadata (e.g., model used, tokens consumed,
-                 tool calls, attachments, etc.)
+        content: The actual message content crossing the boundary (text, JSON, etc.)
+        message_type: Type of boundary crossing (e.g., 'observation', 'action', 'result')
+        time_record: Timing information for the boundary crossing
+        metadata: Boundary crossing metadata. Recommended keys:
+                  - 'step_id': Timestep identifier
+                  - 'from_system_instance_id': UUID of the sending system
+                  - 'to_system_instance_id': UUID of the receiving system
+                  - 'from_system_role': Role of sender (e.g., 'human', 'agent', 'runtime', 'environment')
+                  - 'to_system_role': Role of receiver
+                  - 'boundary_type': Type of Markov blanket boundary being crossed
+                  - 'call_id': Correlate request/response pairs across boundaries
+                  - 'causal_influence': Direction of causal flow
     """
 
     content: str
@@ -70,8 +105,9 @@ class SessionEventMessage:
 class BaseEvent:
     """Base class for all event types.
     
-    This is the foundation for all events in the tracing system. Every event
-    must have a system identifier and timing information.
+    This is the foundation for all events in the tracing system. Every event must
+    have a system identifier and timing information. Events are intra-system facts
+    (they occur within a subsystem) and are not necessarily direct communications.
     
     Attributes:
         system_instance_id: Identifier for the system/component that generated
@@ -95,8 +131,10 @@ class BaseEvent:
 class RuntimeEvent(BaseEvent):
     """Event from runtime system.
     
-    Captures events from the AI system's runtime, typically representing
-    decisions or actions taken by the system.
+    Captures events from the AI system's runtime, typically representing decisions
+    or actions taken by the system (e.g., selecting a tool with arguments).
+    Use paired SessionEventMessages to record the communication of this choice to
+    the environment.
     
     Attributes:
         actions: List of action identifiers or indices. The interpretation
@@ -111,7 +149,9 @@ class RuntimeEvent(BaseEvent):
 class EnvironmentEvent(BaseEvent):
     """Event from environment.
     
-    Captures feedback from the environment in response to system actions.
+    Captures feedback from the environment in response to system actions (e.g.,
+    command output, exit codes, observations). Use a paired SessionEventMessage
+    to record the environment-to-agent communication of the result.
     Follows the Gymnasium/OpenAI Gym convention for compatibility.
     
     Attributes:
@@ -135,6 +175,8 @@ class LMCAISEvent(BaseEvent):
     
     CAIS (Claude AI System) events capture detailed information about LLM calls,
     including performance metrics, cost tracking, and distributed tracing support.
+    Treat provider-specific prompt/completion structures as part of this event's
+    data. Do not emit them as SessionEventMessages.
     
     Attributes:
         model_name: The specific model used (e.g., 'gpt-4', 'claude-3-opus')
@@ -148,6 +190,8 @@ class LMCAISEvent(BaseEvent):
         trace_id: OpenTelemetry compatible trace identifier
         system_state_before: State snapshot before the LLM call
         system_state_after: State snapshot after the LLM call
+        call_records: List of normalized LLM call records capturing request/response
+                      details (messages, tool calls/results, usage, params, etc.).
     """
 
     model_name: str = ""
@@ -161,6 +205,7 @@ class LMCAISEvent(BaseEvent):
     trace_id: Optional[str] = None
     system_state_before: Optional[Dict[str, Any]] = None
     system_state_after: Optional[Dict[str, Any]] = None
+    call_records: List[LLMCallRecord] = field(default_factory=list)
 
 
 @dataclass
@@ -188,7 +233,7 @@ class SessionTimeStep:
     timestamp: datetime = field(default_factory=datetime.utcnow)
     turn_number: Optional[int] = None
     events: List[BaseEvent] = field(default_factory=list)
-    step_messages: List[SessionEventMessage] = field(default_factory=list)
+    markov_blanket_messages: List[SessionEventMarkovBlanketMessage] = field(default_factory=list)
     step_metadata: Dict[str, Any] = field(default_factory=dict)
     completed_at: Optional[datetime] = None
 
@@ -222,7 +267,7 @@ class SessionTrace:
     created_at: datetime = field(default_factory=datetime.utcnow)
     session_time_steps: List[SessionTimeStep] = field(default_factory=list)
     event_history: List[BaseEvent] = field(default_factory=list)
-    message_history: List[SessionEventMessage] = field(default_factory=list)
+    markov_blanket_message_history: List[SessionEventMarkovBlanketMessage] = field(default_factory=list)
     metadata: Dict[str, Any] = field(default_factory=dict)
     session_metadata: Optional[List[Dict[str, Any]]] = None
 

synth_ai/tracing_v3/hooks.py

@@ -37,7 +37,7 @@ from dataclasses import dataclass
 import asyncio
 import inspect
 
-from .abstractions import SessionTrace, SessionTimeStep, BaseEvent, SessionEventMessage
+from .abstractions import SessionTrace, SessionTimeStep, BaseEvent, SessionEventMarkovBlanketMessage
 
 
 @dataclass

synth_ai/tracing_v3/llm_call_record_helpers.py

@@ -0,0 +1,350 @@
+"""Helper functions for creating and populating LLMCallRecord instances.
+
+This module provides utilities to convert vendor responses to LLMCallRecord
+format and compute aggregates from call records.
+"""
+
+import uuid
+import json
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+from synth_ai.tracing_v3.lm_call_record_abstractions import (
+    LLMCallRecord,
+    LLMUsage,
+    LLMRequestParams,
+    LLMMessage,
+    LLMContentPart,
+    ToolCallSpec,
+    ToolCallResult,
+    LLMChunk,
+)
+from synth_ai.lm.vendors.base import BaseLMResponse
+
+
+def create_llm_call_record_from_response(
+    response: BaseLMResponse,
+    model_name: str,
+    provider: str,
+    messages: List[Dict[str, Any]],
+    temperature: float = 0.8,
+    request_params: Optional[Dict[str, Any]] = None,
+    tools: Optional[List] = None,
+    started_at: Optional[datetime] = None,
+    completed_at: Optional[datetime] = None,
+    latency_ms: Optional[int] = None,
+) -> LLMCallRecord:
+    """Create an LLMCallRecord from a vendor response.
+    
+    Args:
+        response: The vendor response object
+        model_name: Name of the model used
+        provider: Provider name (e.g., 'openai', 'anthropic')
+        messages: Input messages sent to the model
+        temperature: Temperature parameter used
+        request_params: Additional request parameters
+        tools: Tools provided to the model
+        started_at: When the request started
+        completed_at: When the request completed
+        latency_ms: End-to-end latency in milliseconds
+    
+    Returns:
+        A populated LLMCallRecord instance
+    """
+    # Generate call ID
+    call_id = str(uuid.uuid4())
+    
+    # Determine API type from response
+    api_type = "chat_completions"  # Default
+    if hasattr(response, 'api_type'):
+        if response.api_type == "responses":
+            api_type = "responses"
+        elif response.api_type == "completions":
+            api_type = "completions"
+    
+    # Convert input messages to LLMMessage format
+    input_messages = []
+    for msg in messages:
+        role = msg.get("role", "user")
+        content = msg.get("content", "")
+        
+        # Handle different content formats
+        if isinstance(content, str):
+            parts = [LLMContentPart(type="text", text=content)]
+        elif isinstance(content, list):
+            parts = []
+            for item in content:
+                if isinstance(item, dict):
+                    if item.get("type") == "text":
+                        parts.append(LLMContentPart(type="text", text=item.get("text", "")))
+                    elif item.get("type") == "image_url":
+                        parts.append(LLMContentPart(
+                            type="image",
+                            uri=item.get("image_url", {}).get("url", ""),
+                            mime_type="image/jpeg"
+                        ))
+                    elif item.get("type") == "image":
+                        parts.append(LLMContentPart(
+                            type="image",
+                            data=item.get("source", {}),
+                            mime_type=item.get("source", {}).get("media_type", "image/jpeg")
+                        ))
+                else:
+                    parts.append(LLMContentPart(type="text", text=str(item)))
+        else:
+            parts = [LLMContentPart(type="text", text=str(content))]
+        
+        input_messages.append(LLMMessage(role=role, parts=parts))
+    
+    # Extract output messages from response
+    output_messages = []
+    output_text = None
+    
+    if hasattr(response, 'raw_response'):
+        # Extract assistant message
+        output_text = response.raw_response
+        output_messages.append(
+            LLMMessage(
+                role="assistant",
+                parts=[LLMContentPart(type="text", text=output_text)]
+            )
+        )
+    
+    # Extract tool calls if present
+    output_tool_calls = []
+    if hasattr(response, 'tool_calls') and response.tool_calls:
+        for idx, tool_call in enumerate(response.tool_calls):
+            if isinstance(tool_call, dict):
+                output_tool_calls.append(
+                    ToolCallSpec(
+                        name=tool_call.get("function", {}).get("name", ""),
+                        arguments_json=tool_call.get("function", {}).get("arguments", "{}"),
+                        call_id=tool_call.get("id", f"tool_{idx}"),
+                        index=idx
+                    )
+                )
+    
+    # Extract usage information
+    usage = None
+    if hasattr(response, 'usage') and response.usage:
+        usage = LLMUsage(
+            input_tokens=response.usage.get("input_tokens"),
+            output_tokens=response.usage.get("output_tokens"),
+            total_tokens=response.usage.get("total_tokens"),
+            cost_usd=response.usage.get("cost_usd"),
+            # Additional token accounting if available
+            reasoning_tokens=response.usage.get("reasoning_tokens"),
+            reasoning_input_tokens=response.usage.get("reasoning_input_tokens"),
+            reasoning_output_tokens=response.usage.get("reasoning_output_tokens"),
+            cache_write_tokens=response.usage.get("cache_write_tokens"),
+            cache_read_tokens=response.usage.get("cache_read_tokens"),
+        )
+    
+    # Build request parameters
+    params = LLMRequestParams(
+        temperature=temperature,
+        top_p=request_params.get("top_p") if request_params else None,
+        max_tokens=request_params.get("max_tokens") if request_params else None,
+        stop=request_params.get("stop") if request_params else None,
+        raw_params=request_params or {}
+    )
+    
+    # Handle response-specific fields
+    finish_reason = None
+    if hasattr(response, 'finish_reason'):
+        finish_reason = response.finish_reason
+    elif hasattr(response, 'stop_reason'):
+        finish_reason = response.stop_reason
+    
+    # Create the call record
+    record = LLMCallRecord(
+        call_id=call_id,
+        api_type=api_type,
+        provider=provider,
+        model_name=model_name,
+        started_at=started_at or datetime.utcnow(),
+        completed_at=completed_at or datetime.utcnow(),
+        latency_ms=latency_ms,
+        request_params=params,
+        input_messages=input_messages,
+        input_text=None,  # For completions API
+        tool_choice="auto" if tools else None,
+        output_messages=output_messages,
+        output_text=output_text,
+        output_tool_calls=output_tool_calls,
+        usage=usage,
+        finish_reason=finish_reason,
+        outcome="success",
+        metadata={
+            "has_tools": tools is not None,
+            "num_tools": len(tools) if tools else 0,
+        }
+    )
+    
+    # Store response ID if available (for Responses API)
+    if hasattr(response, 'response_id') and response.response_id:
+        record.metadata["response_id"] = response.response_id
+        record.provider_request_id = response.response_id
+    
+    return record
+
+
+def compute_aggregates_from_call_records(call_records: List[LLMCallRecord]) -> Dict[str, Any]:
+    """Compute aggregate statistics from a list of LLMCallRecord instances.
+    
+    Args:
+        call_records: List of LLMCallRecord instances
+    
+    Returns:
+        Dictionary containing aggregated statistics
+    """
+    aggregates = {
+        "input_tokens": 0,
+        "output_tokens": 0,
+        "total_tokens": 0,
+        "reasoning_tokens": 0,
+        "cost_usd": 0.0,
+        "latency_ms": 0,
+        "models_used": set(),
+        "providers_used": set(),
+        "tool_calls_count": 0,
+        "error_count": 0,
+        "success_count": 0,
+        "call_count": len(call_records)
+    }
+    
+    for record in call_records:
+        # Token aggregation
+        if record.usage:
+            if record.usage.input_tokens:
+                aggregates["input_tokens"] += record.usage.input_tokens
+            if record.usage.output_tokens:
+                aggregates["output_tokens"] += record.usage.output_tokens
+            if record.usage.total_tokens:
+                aggregates["total_tokens"] += record.usage.total_tokens
+            if record.usage.reasoning_tokens:
+                aggregates["reasoning_tokens"] += record.usage.reasoning_tokens
+            if record.usage.cost_usd:
+                aggregates["cost_usd"] += record.usage.cost_usd
+        
+        # Latency aggregation
+        if record.latency_ms:
+            aggregates["latency_ms"] += record.latency_ms
+        
+        # Model and provider tracking
+        if record.model_name:
+            aggregates["models_used"].add(record.model_name)
+        if record.provider:
+            aggregates["providers_used"].add(record.provider)
+        
+        # Tool calls
+        aggregates["tool_calls_count"] += len(record.output_tool_calls)
+        
+        # Success/error tracking
+        if record.outcome == "error":
+            aggregates["error_count"] += 1
+        elif record.outcome == "success":
+            aggregates["success_count"] += 1
+    
+    # Convert sets to lists for JSON serialization
+    aggregates["models_used"] = list(aggregates["models_used"])
+    aggregates["providers_used"] = list(aggregates["providers_used"])
+    
+    # Compute averages
+    if aggregates["call_count"] > 0:
+        aggregates["avg_latency_ms"] = aggregates["latency_ms"] / aggregates["call_count"]
+        aggregates["avg_input_tokens"] = aggregates["input_tokens"] / aggregates["call_count"]
+        aggregates["avg_output_tokens"] = aggregates["output_tokens"] / aggregates["call_count"]
+    
+    return aggregates
+
+
+def create_llm_call_record_from_streaming(
+    chunks: List[LLMChunk],
+    model_name: str,
+    provider: str,
+    messages: List[Dict[str, Any]],
+    temperature: float = 0.8,
+    request_params: Optional[Dict[str, Any]] = None,
+    started_at: Optional[datetime] = None,
+    completed_at: Optional[datetime] = None,
+) -> LLMCallRecord:
+    """Create an LLMCallRecord from streaming chunks.
+    
+    This function reconstructs a complete LLMCallRecord from streaming
+    response chunks, useful for Responses API or streaming Chat Completions.
+    
+    Args:
+        chunks: List of LLMChunk instances from streaming
+        model_name: Name of the model used
+        provider: Provider name
+        messages: Input messages sent to the model
+        temperature: Temperature parameter used
+        request_params: Additional request parameters
+        started_at: When the request started
+        completed_at: When the request completed
+    
+    Returns:
+        A populated LLMCallRecord instance
+    """
+    # Reconstruct output text from chunks
+    output_text = "".join(
+        chunk.delta_text for chunk in chunks 
+        if chunk.delta_text
+    )
+    
+    # Calculate latency from chunk timestamps
+    latency_ms = None
+    if chunks and started_at:
+        last_chunk_time = chunks[-1].received_at
+        latency_ms = int((last_chunk_time - started_at).total_seconds() * 1000)
+    
+    # Convert input messages
+    input_messages = []
+    for msg in messages:
+        role = msg.get("role", "user")
+        content = msg.get("content", "")
+        
+        if isinstance(content, str):
+            parts = [LLMContentPart(type="text", text=content)]
+        else:
+            parts = [LLMContentPart(type="text", text=str(content))]
+        
+        input_messages.append(LLMMessage(role=role, parts=parts))
+    
+    # Create output message
+    output_messages = [
+        LLMMessage(
+            role="assistant",
+            parts=[LLMContentPart(type="text", text=output_text)]
+        )
+    ]
+    
+    # Build request parameters
+    params = LLMRequestParams(
+        temperature=temperature,
+        raw_params=request_params or {}
+    )
+    
+    # Create the call record
+    record = LLMCallRecord(
+        call_id=str(uuid.uuid4()),
+        api_type="responses",  # Streaming typically from Responses API
+        provider=provider,
+        model_name=model_name,
+        started_at=started_at or datetime.utcnow(),
+        completed_at=completed_at or datetime.utcnow(),
+        latency_ms=latency_ms,
+        request_params=params,
+        input_messages=input_messages,
+        output_messages=output_messages,
+        output_text=output_text,
+        chunks=chunks,
+        outcome="success",
+        metadata={
+            "chunk_count": len(chunks),
+            "streaming": True
+        }
+    )
+    
+    return record