mseep-agentops 0.4.18__py3-none-any.whl → 0.4.23__py3-none-any.whl

agentops/instrumentation/agentic/openai_agents/attributes/completion.py

@@ -0,0 +1,172 @@
+"""Completion processing utilities for OpenAI Agents instrumentation.
+
+This module handles completion content processing from both the Chat Completions API
+and the OpenAI Response API formats, extracting messages, tool calls, function calls, etc.
+"""
+
+from typing import Any, Dict
+
+from agentops.instrumentation.common.attributes import AttributeMap
+
+from agentops.logging import logger
+from agentops.helpers.serialization import model_to_dict
+from agentops.semconv import (
+    SpanAttributes,
+    MessageAttributes,
+)
+from agentops.instrumentation.agentic.openai_agents.attributes.tokens import process_token_usage
+
+
+def get_generation_output_attributes(output: Any) -> Dict[str, Any]:
+    """Extract LLM response attributes from an `openai/completions` object.
+
+    Args:
+        output: The response object (can be dict, Response object, or other format)
+
+    Returns:
+        Dictionary of attributes extracted from the response in a consistent format
+    """
+    # Convert model to dictionary for easier processing
+    response_dict = model_to_dict(output)
+    result: AttributeMap = {}
+
+    if not response_dict:
+        # Handle output as string if it's not a dict
+        if isinstance(output, str):
+            # For string output, just return the minimal set of attributes
+            return {}
+        return result
+
+    # Check for OpenAI Agents SDK response format (has raw_responses array)
+    if "raw_responses" in response_dict and isinstance(response_dict["raw_responses"], list):
+        result.update(get_raw_response_attributes(response_dict))
+    else:
+        # TODO base attributes for completion type
+
+        # Get completions or response API output attributes first
+        if "choices" in response_dict:
+            result.update(get_chat_completions_attributes(response_dict))
+
+        # Extract token usage from dictionary for standard formats
+        usage_attributes: AttributeMap = {}
+        if "usage" in response_dict:
+            process_token_usage(response_dict["usage"], usage_attributes)
+            result.update(usage_attributes)
+
+        # Extract token usage from Response object directly if dict conversion didn't work
+        if hasattr(output, "usage") and output.usage:
+            direct_usage_attributes: AttributeMap = {}
+            process_token_usage(output.usage, direct_usage_attributes)
+            result.update(direct_usage_attributes)
+
+    return result
+
+
+def get_raw_response_attributes(response: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract attributes from OpenAI Agents SDK response format (with raw_responses).
+
+    This function handles the specific structure of OpenAI Agents SDK responses,
+    which include a raw_responses array containing the actual API responses.
+    This is the format used specifically by the Agents SDK, not the standard OpenAI API.
+
+    Args:
+        response: The OpenAI Agents SDK response dictionary (containing raw_responses array)
+
+    Returns:
+        Dictionary of attributes extracted from the Agents SDK response
+    """
+    result: AttributeMap = {}
+
+    # Set the LLM system to OpenAI
+    result[SpanAttributes.LLM_SYSTEM] = "openai"
+
+    # Process raw responses
+    if "raw_responses" in response and isinstance(response["raw_responses"], list):
+        for i, raw_response in enumerate(response["raw_responses"]):
+            # Extract token usage from the first raw response
+            if "usage" in raw_response and isinstance(raw_response["usage"], dict):
+                usage_attrs: AttributeMap = {}
+                process_token_usage(raw_response["usage"], usage_attrs)
+                result.update(usage_attrs)
+                logger.debug(f"Extracted token usage from raw_responses[{i}]: {usage_attrs}")
+
+            # Extract output content
+            if "output" in raw_response and isinstance(raw_response["output"], list):
+                for j, output_item in enumerate(raw_response["output"]):
+                    # Process message content
+                    if "content" in output_item and isinstance(output_item["content"], list):
+                        for content_item in output_item["content"]:
+                            if content_item.get("type") == "output_text" and "text" in content_item:
+                                # Set message content attribute using the standard convention
+                                result[MessageAttributes.COMPLETION_CONTENT.format(i=j)] = content_item["text"]
+
+                    # Process role
+                    if "role" in output_item:
+                        result[MessageAttributes.COMPLETION_ROLE.format(i=j)] = output_item["role"]
+
+                    # Process tool calls
+                    if "tool_calls" in output_item and isinstance(output_item["tool_calls"], list):
+                        for k, tool_call in enumerate(output_item["tool_calls"]):
+                            tool_id = tool_call.get("id", "")
+                            # Handle function format
+                            if "function" in tool_call and isinstance(tool_call["function"], dict):
+                                function = tool_call["function"]
+                                result[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=j, j=k)] = tool_id
+                                result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=j, j=k)] = function.get(
+                                    "name", ""
+                                )
+                                result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=j, j=k)] = (
+                                    function.get("arguments", "")
+                                )
+
+    return result
+
+
+def get_chat_completions_attributes(response: Dict[str, Any]) -> Dict[str, Any]:
+    """Get attributes from OpenAI Chat Completions API format (with choices array).
+
+    This function specifically handles the original Chat Completions API format
+    that uses a 'choices' array with 'message' objects, as opposed to the newer
+    Response API format that uses an 'output' array.
+
+    Args:
+        response: The response dictionary containing chat completions (with choices array)
+
+    Returns:
+        Dictionary of chat completion attributes
+    """
+    result: AttributeMap = {}
+
+    if "choices" not in response:
+        return result
+
+    for i, choice in enumerate(response["choices"]):
+        if "finish_reason" in choice:
+            result[MessageAttributes.COMPLETION_FINISH_REASON.format(i=i)] = choice["finish_reason"]
+
+        message = choice.get("message", {})
+
+        if "role" in message:
+            result[MessageAttributes.COMPLETION_ROLE.format(i=i)] = message["role"]
+
+        if "content" in message:
+            content = message["content"] if message["content"] is not None else ""
+            result[MessageAttributes.COMPLETION_CONTENT.format(i=i)] = content
+
+        if "tool_calls" in message and message["tool_calls"] is not None:
+            tool_calls = message["tool_calls"]
+            for j, tool_call in enumerate(tool_calls):
+                if "function" in tool_call:
+                    function = tool_call["function"]
+                    result[MessageAttributes.COMPLETION_TOOL_CALL_ID.format(i=i, j=j)] = tool_call.get("id")
+                    result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=i, j=j)] = function.get("name")
+                    result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=i, j=j)] = function.get(
+                        "arguments"
+                    )
+
+        if "function_call" in message and message["function_call"] is not None:
+            function_call = message["function_call"]
+            result[MessageAttributes.COMPLETION_TOOL_CALL_NAME.format(i=i)] = function_call.get("name")
+            result[MessageAttributes.COMPLETION_TOOL_CALL_ARGUMENTS.format(i=i)] = function_call.get("arguments")
+
+    return result

agentops/instrumentation/agentic/openai_agents/attributes/model.py

@@ -0,0 +1,58 @@
+"""Model information extraction for OpenAI Agents instrumentation.
+
+This module provides utilities for extracting model information and parameters
+from various object types, centralizing model attribute handling logic.
+"""
+
+from typing import Any, Dict
+from agentops.semconv import SpanAttributes
+from agentops.instrumentation.common.attributes import AttributeMap, _extract_attributes_from_mapping
+
+
+# Parameter mapping dictionary for model parameters
+MODEL_CONFIG_ATTRIBUTES: AttributeMap = {
+    SpanAttributes.LLM_REQUEST_TEMPERATURE: "temperature",
+    SpanAttributes.LLM_REQUEST_TOP_P: "top_p",
+    SpanAttributes.LLM_REQUEST_FREQUENCY_PENALTY: "frequency_penalty",
+    SpanAttributes.LLM_REQUEST_PRESENCE_PENALTY: "presence_penalty",
+    SpanAttributes.LLM_REQUEST_MAX_TOKENS: "max_tokens",
+    # TODO we need to establish semantic conventions for the following:
+    # tool_choice
+    # parallel_tool_calls
+    # truncation
+    # store
+    # language
+    # prompt
+    # turn_detection
+    SpanAttributes.LLM_REQUEST_INSTRUCTIONS: "instructions",
+    SpanAttributes.LLM_REQUEST_VOICE: "voice",
+    SpanAttributes.LLM_REQUEST_SPEED: "speed",
+}
+
+
+def get_model_attributes(model_name: str) -> Dict[str, Any]:
+    """Get model name attributes for both request and response for consistency.
+
+    Args:
+        model_name: The model name to set
+
+    Returns:
+        Dictionary of model name attributes
+    """
+    return {
+        SpanAttributes.LLM_REQUEST_MODEL: model_name,
+        SpanAttributes.LLM_RESPONSE_MODEL: model_name,
+        SpanAttributes.LLM_SYSTEM: "openai",
+    }
+
+
+def get_model_config_attributes(model_config: Any) -> Dict[str, Any]:
+    """Extract model configuration attributes using the model parameter mapping.
+
+    Args:
+        model_config: The model configuration object
+
+    Returns:
+        Dictionary of extracted model configuration attributes
+    """
+    return _extract_attributes_from_mapping(model_config, MODEL_CONFIG_ATTRIBUTES)

agentops/instrumentation/agentic/openai_agents/attributes/tokens.py

@@ -0,0 +1,275 @@
+"""Token processing and metrics for the OpenAI Agents instrumentation.
+
+This module contains functions for processing token usage data from OpenAI responses,
+including standardized handling of different API formats (Chat Completions API vs Response API)
+and recording token usage metrics.
+"""
+
+import json
+from typing import Any, Dict, Optional
+
+from agentops.semconv import SpanAttributes
+from agentops.logging import logger
+
+
+def safe_parse(content: str) -> Optional[Dict[str, Any]]:
+    """Safely parse JSON content from a string.
+
+    Args:
+        content: String content that might contain JSON
+
+    Returns:
+        Parsed dictionary if content is valid JSON, None otherwise
+    """
+    if not isinstance(content, str):
+        return None
+
+    try:
+        # Try to parse the string as JSON
+        return json.loads(content)
+    except (json.JSONDecodeError, TypeError, ValueError):
+        # If parsing fails, log a debug message and return None
+        logger.debug(f"Failed to parse JSON content: {content[:100]}...")
+        return None
+
+
+def extract_nested_usage(content: Any) -> Optional[Dict[str, Any]]:
+    """Recursively extract usage data from potentially nested response structures.
+
+    Handles multiple nesting patterns:
+    1. Direct usage field at the top level
+    2. Usage nested in completion content JSON string
+    3. Usage nested in response.output[].content[].text
+
+    Args:
+        content: Any content object that might contain usage data
+
+    Returns:
+        Extracted usage dictionary or None if not found
+    """
+    # Case: direct dictionary with usage field
+    if isinstance(content, dict) and "usage" in content:
+        return content["usage"]
+
+    # Case: JSON string that might contain usage
+    if isinstance(content, str):
+        parsed_data = safe_parse(content)
+        if parsed_data:
+            # Direct usage field in parsed JSON
+            if "usage" in parsed_data and isinstance(parsed_data["usage"], dict):
+                return parsed_data["usage"]
+
+            # Response API format with nested output structure
+            if "output" in parsed_data and isinstance(parsed_data["output"], list):
+                # Usage at top level in Response format
+                if "usage" in parsed_data:
+                    return parsed_data["usage"]
+
+    # Case: complex nested structure with output array
+    # This handles the Response API format where usage is at the top level
+    if isinstance(content, dict):
+        if "output" in content and isinstance(content["output"], list):
+            if "usage" in content:
+                return content["usage"]
+
+    return None
+
+
+def process_token_usage(
+    usage: Dict[str, Any], attributes: Dict[str, Any], completion_content: Optional[str] = None
+) -> Dict[str, Any]:
+    """Process token usage data from OpenAI responses using standardized attribute naming.
+
+    Args:
+        usage: Dictionary containing token usage data
+        attributes: Dictionary where attributes will be set
+        completion_content: Optional JSON string that may contain token usage info
+
+    Returns:
+        Dictionary mapping token types to counts for metrics
+    """
+    # Result dictionary for metric recording
+    result = {}
+
+    # If usage is empty or None, use completion_content to find usage data
+    if not usage or (isinstance(usage, dict) and len(usage) == 0):
+        if completion_content:
+            logger.debug("TOKENS: Usage is empty, trying to extract from completion content")
+            extracted_usage = extract_nested_usage(completion_content)
+            if extracted_usage:
+                usage = extracted_usage
+
+    # Always set token usage attributes directly on the span to ensure they're captured
+    # For both Chat Completions API and Response API formats
+
+    # Helper to get an attribute from either a dict or an object
+    def get_value(obj, key):
+        if isinstance(obj, dict) and key in obj:
+            return obj[key]
+        elif hasattr(obj, key):
+            return getattr(obj, key)
+        return None
+
+    # Helper to check if an object has an attribute
+    def has_key(obj, key):
+        if isinstance(obj, dict):
+            return key in obj
+        return hasattr(obj, key)
+
+    # Process prompt/input tokens
+    if has_key(usage, "prompt_tokens"):
+        prompt_tokens = get_value(usage, "prompt_tokens")
+        attributes[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] = prompt_tokens
+        result["prompt_tokens"] = prompt_tokens
+    elif has_key(usage, "input_tokens"):
+        input_tokens = get_value(usage, "input_tokens")
+        attributes[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] = input_tokens
+        result["prompt_tokens"] = input_tokens
+
+    # Process completion/output tokens
+    if has_key(usage, "completion_tokens"):
+        completion_tokens = get_value(usage, "completion_tokens")
+        attributes[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] = completion_tokens
+        result["completion_tokens"] = completion_tokens
+    elif has_key(usage, "output_tokens"):
+        output_tokens = get_value(usage, "output_tokens")
+        attributes[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] = output_tokens
+        result["completion_tokens"] = output_tokens
+
+    # Process total tokens
+    if has_key(usage, "total_tokens"):
+        total_tokens = get_value(usage, "total_tokens")
+        attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] = total_tokens
+        result["total_tokens"] = total_tokens
+
+    # Process Response API specific token details using defined semantic conventions
+
+    # Process reasoning tokens (from Response API output_tokens_details)
+    output_tokens_details = None
+    if has_key(usage, "output_tokens_details"):
+        output_tokens_details = get_value(usage, "output_tokens_details")
+
+    if output_tokens_details:
+        # Handle both dict and object types
+        if isinstance(output_tokens_details, dict):
+            details = output_tokens_details
+            if "reasoning_tokens" in details:
+                attributes[SpanAttributes.LLM_USAGE_REASONING_TOKENS] = details["reasoning_tokens"]
+                result["reasoning_tokens"] = details["reasoning_tokens"]
+        elif hasattr(output_tokens_details, "reasoning_tokens"):
+            reasoning_tokens = output_tokens_details.reasoning_tokens
+            attributes[SpanAttributes.LLM_USAGE_REASONING_TOKENS] = reasoning_tokens
+            result["reasoning_tokens"] = reasoning_tokens
+
+    # Process cached tokens (from Response API input_tokens_details)
+    input_tokens_details = None
+    if has_key(usage, "input_tokens_details"):
+        input_tokens_details = get_value(usage, "input_tokens_details")
+
+    if input_tokens_details:
+        # Handle both dict and object types
+        if isinstance(input_tokens_details, dict):
+            details = input_tokens_details
+            if "cached_tokens" in details:
+                attributes[SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS] = details["cached_tokens"]
+                result["cached_input_tokens"] = details["cached_tokens"]
+        # Handle object with cached_tokens attribute
+        elif hasattr(input_tokens_details, "cached_tokens"):
+            cached_tokens = input_tokens_details.cached_tokens
+            attributes[SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS] = cached_tokens
+            result["cached_input_tokens"] = cached_tokens
+
+    # Log all token-related attributes that were set
+    token_attrs = {k: v for k, v in attributes.items() if k.startswith("gen_ai.usage")}
+
+    # If we still have no token attributes, try one more approach - look for nested output structure
+    if not token_attrs and completion_content:
+        try:
+            # Parse the completion content to see if we can find more deeply nested usage data
+            parsed_content = safe_parse(completion_content)
+            if parsed_content and isinstance(parsed_content, dict):
+                # If this is a Response API format, check for nested output structure
+                if "output" in parsed_content and isinstance(parsed_content["output"], list):
+                    for output_item in parsed_content["output"]:
+                        # Check if this has nested content with usage
+                        if "content" in output_item and isinstance(output_item["content"], list):
+                            for content_item in output_item["content"]:
+                                if "text" in content_item:
+                                    # Try to parse this text for usage data
+                                    parsed_text = safe_parse(content_item["text"])
+                                    if parsed_text and "usage" in parsed_text:
+                                        logger.debug(f"Found deeply nested usage data: {parsed_text['usage']}")
+                                        # Process this usage data recursively
+                                        return process_token_usage(parsed_text["usage"], attributes)
+        except Exception as e:
+            logger.debug(f"Error during deep token extraction: {e}")
+
+    return result
+
+
+def map_token_type_to_metric_name(token_type: str) -> str:
+    """Maps token type names from SpanAttributes to simplified metric names.
+
+    Args:
+        token_type: Token type name, could be a full semantic convention or a simple name
+
+    Returns:
+        Simplified token type name for metrics
+    """
+    # If token_type is a semantic convention (contains a dot), extract the last part
+    if isinstance(token_type, str) and "." in token_type:
+        parts = token_type.split(".")
+        token_type = parts[-1]
+
+    # Map to simplified metric names
+    if token_type == "prompt_tokens":
+        return "input"
+    elif token_type == "completion_tokens":
+        return "output"
+    elif token_type == "reasoning_tokens":
+        return "reasoning"
+
+    # Return as-is if no mapping needed
+    return token_type
+
+
+def get_token_metric_attributes(usage: Dict[str, Any], model_name: str) -> Dict[str, Dict[str, Any]]:
+    """Get token usage metric attributes from usage data.
+
+    Args:
+        usage: Dictionary containing token usage data
+        model_name: Name of the model used
+
+    Returns:
+        Dictionary mapping token types to metric data including value and attributes
+    """
+    # Process all token types using our standardized processor
+    token_counts = process_token_usage(usage, {})
+
+    # Common attributes for all metrics
+    common_attributes = {
+        "model": model_name,
+        SpanAttributes.LLM_REQUEST_MODEL: model_name,
+        SpanAttributes.LLM_SYSTEM: "openai",
+    }
+
+    # Prepare metrics data for each token type
+    metrics_data = {}
+    for token_type, count in token_counts.items():
+        # Skip if no count
+        if not count:
+            continue
+
+        # Map token type to simplified metric name
+        metric_token_type = map_token_type_to_metric_name(token_type)
+
+        # Prepare the metric data
+        metrics_data[token_type] = {
+            "value": count,
+            "attributes": {
+                "token_type": metric_token_type,
+                **common_attributes,
+            },
+        }
+
+    return metrics_data