synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/tracing_v3/llm_call_record_helpers.py

@@ -1,391 +0,0 @@
-"""Helper functions for creating and populating LLMCallRecord instances.
-
-This module provides utilities to convert vendor responses to LLMCallRecord
-format and compute aggregates from call records.
-"""
-
-from __future__ import annotations
-
-import uuid
-from dataclasses import dataclass, field
-from datetime import datetime, timezone
-from typing import Any, TypedDict, cast
-
-from .lm_call_record_abstractions import (
-    LLMCallRecord,
-    LLMChunk,
-    LLMContentPart,
-    LLMMessage,
-    LLMRequestParams,
-    LLMUsage,
-    ToolCallSpec,
-)
-
-BaseLMResponse = Any
-
-
-class _UsageDict(TypedDict, total=False):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-    reasoning_tokens: int
-    cost_usd: float
-    duration_ms: int
-    reasoning_input_tokens: int
-    reasoning_output_tokens: int
-    cache_write_tokens: int
-    cache_read_tokens: int
-
-
-def create_llm_call_record_from_response(
-    response: BaseLMResponse,
-    model_name: str,
-    provider: str,
-    messages: list[dict[str, Any]],
-    temperature: float = 0.8,
-    request_params: dict[str, Any] | None = None,
-    tools: list | None = None,
-    started_at: datetime | None = None,
-    completed_at: datetime | None = None,
-    latency_ms: int | None = None,
-) -> LLMCallRecord:
-    """Create an LLMCallRecord from a vendor response.
-
-    Args:
-        response: The vendor response object
-        model_name: Name of the model used
-        provider: Provider name (e.g., 'openai', 'anthropic')
-        messages: Input messages sent to the model
-        temperature: Temperature parameter used
-        request_params: Additional request parameters
-        tools: Tools provided to the model
-        started_at: When the request started
-        completed_at: When the request completed
-        latency_ms: End-to-end latency in milliseconds
-
-    Returns:
-        A populated LLMCallRecord instance
-    """
-    # Generate call ID
-    call_id = str(uuid.uuid4())
-
-    # Determine API type from response
-    api_type = "chat_completions"  # Default
-    if hasattr(response, "api_type"):
-        if response.api_type == "responses":
-            api_type = "responses"
-        elif response.api_type == "completions":
-            api_type = "completions"
-
-    # Convert input messages to LLMMessage format
-    input_messages = []
-    for msg in messages:
-        role = msg.get("role", "user")
-        content = msg.get("content", "")
-
-        # Handle different content formats
-        if isinstance(content, str):
-            parts = [LLMContentPart(type="text", text=content)]
-        elif isinstance(content, list):
-            parts = []
-            for item in content:
-                if isinstance(item, dict):
-                    if item.get("type") == "text":
-                        parts.append(LLMContentPart(type="text", text=item.get("text", "")))
-                    elif item.get("type") == "image_url":
-                        parts.append(
-                            LLMContentPart(
-                                type="image",
-                                uri=item.get("image_url", {}).get("url", ""),
-                                mime_type="image/jpeg",
-                            )
-                        )
-                    elif item.get("type") == "image":
-                        parts.append(
-                            LLMContentPart(
-                                type="image",
-                                data=item.get("source", {}),
-                                mime_type=item.get("source", {}).get("media_type", "image/jpeg"),
-                            )
-                        )
-                else:
-                    parts.append(LLMContentPart(type="text", text=str(item)))
-        else:
-            parts = [LLMContentPart(type="text", text=str(content))]
-
-        input_messages.append(LLMMessage(role=role, parts=parts))
-
-    # Extract output messages from response
-    output_messages = []
-    output_text = None
-
-    if hasattr(response, "raw_response"):
-        # Extract assistant message
-        output_text = response.raw_response
-        output_messages.append(
-            LLMMessage(role="assistant", parts=[LLMContentPart(type="text", text=output_text)])
-        )
-
-    # Extract tool calls if present
-    output_tool_calls: list[ToolCallSpec] = []
-    tool_calls_data = cast(list[dict[str, Any]] | None, getattr(response, "tool_calls", None))
-    if tool_calls_data:
-        for idx, tool_call in enumerate(tool_calls_data):
-            if isinstance(tool_call, dict):
-                output_tool_calls.append(
-                    ToolCallSpec(
-                        name=tool_call.get("function", {}).get("name", ""),
-                        arguments_json=tool_call.get("function", {}).get("arguments", "{}"),
-                        call_id=tool_call.get("id", f"tool_{idx}"),
-                        index=idx,
-                    )
-                )
-
-    # Extract usage information
-    usage = None
-    usage_data = cast(_UsageDict | None, getattr(response, "usage", None))
-    if usage_data:
-        usage = LLMUsage(
-            input_tokens=usage_data.get("input_tokens"),
-            output_tokens=usage_data.get("output_tokens"),
-            total_tokens=usage_data.get("total_tokens"),
-            cost_usd=usage_data.get("cost_usd"),
-            # Additional token accounting if available
-            reasoning_tokens=usage_data.get("reasoning_tokens"),
-            reasoning_input_tokens=usage_data.get("reasoning_input_tokens"),
-            reasoning_output_tokens=usage_data.get("reasoning_output_tokens"),
-            cache_write_tokens=usage_data.get("cache_write_tokens"),
-            cache_read_tokens=usage_data.get("cache_read_tokens"),
-        )
-
-    # Build request parameters
-    params = LLMRequestParams(
-        temperature=temperature,
-        top_p=request_params.get("top_p") if request_params else None,
-        max_tokens=request_params.get("max_tokens") if request_params else None,
-        stop=request_params.get("stop") if request_params else None,
-        raw_params=request_params or {},
-    )
-
-    # Handle response-specific fields
-    finish_reason = None
-    if hasattr(response, "finish_reason"):
-        finish_reason = response.finish_reason
-    elif hasattr(response, "stop_reason"):
-        finish_reason = response.stop_reason
-
-    # Create the call record
-    record = LLMCallRecord(
-        call_id=call_id,
-        api_type=api_type,
-        provider=provider,
-        model_name=model_name,
-        started_at=started_at or datetime.now(timezone.utc),
-        completed_at=completed_at or datetime.now(timezone.utc),
-        latency_ms=latency_ms,
-        request_params=params,
-        input_messages=input_messages,
-        input_text=None,  # For completions API
-        tool_choice="auto" if tools else None,
-        output_messages=output_messages,
-        output_text=output_text,
-        output_tool_calls=output_tool_calls,
-        usage=usage,
-        finish_reason=finish_reason,
-        outcome="success",
-        metadata={
-            "has_tools": tools is not None,
-            "num_tools": len(tools) if tools else 0,
-        },
-    )
-
-    # Store response ID if available (for Responses API)
-    if hasattr(response, "response_id") and response.response_id:
-        record.metadata["response_id"] = response.response_id
-        record.provider_request_id = response.response_id
-
-    return record
-
-
-@dataclass
-class _AggregateAccumulator:
-    """Mutable accumulator for call record aggregates."""
-
-    call_count: int = 0
-    input_tokens: int = 0
-    output_tokens: int = 0
-    total_tokens: int = 0
-    reasoning_tokens: int = 0
-    cost_usd: float = 0.0
-    latency_ms: int = 0
-    models_used: set[str] = field(default_factory=set)
-    providers_used: set[str] = field(default_factory=set)
-    tool_calls_count: int = 0
-    error_count: int = 0
-    success_count: int = 0
-
-
-class AggregateSummary(TypedDict, total=False):
-    """Aggregate metrics derived from call records."""
-
-    call_count: int
-    input_tokens: int
-    output_tokens: int
-    total_tokens: int
-    reasoning_tokens: int
-    cost_usd: float
-    latency_ms: int
-    models_used: list[str]
-    providers_used: list[str]
-    tool_calls_count: int
-    error_count: int
-    success_count: int
-    avg_latency_ms: float
-    avg_input_tokens: float
-    avg_output_tokens: float
-
-
-def compute_aggregates_from_call_records(call_records: list[LLMCallRecord]) -> AggregateSummary:
-    """Compute aggregate statistics from a list of LLMCallRecord instances.
-
-    Args:
-        call_records: List of LLMCallRecord instances
-
-    Returns:
-        Dictionary containing aggregated statistics
-    """
-    aggregates = _AggregateAccumulator(call_count=len(call_records))
-
-    for record in call_records:
-        # Token aggregation
-        if record.usage:
-            if record.usage.input_tokens:
-                aggregates.input_tokens += record.usage.input_tokens
-            if record.usage.output_tokens:
-                aggregates.output_tokens += record.usage.output_tokens
-            if record.usage.total_tokens:
-                aggregates.total_tokens += record.usage.total_tokens
-            if record.usage.reasoning_tokens:
-                aggregates.reasoning_tokens += record.usage.reasoning_tokens
-            if record.usage.cost_usd:
-                aggregates.cost_usd += record.usage.cost_usd
-
-        # Latency aggregation
-        if record.latency_ms is not None:
-            aggregates.latency_ms += record.latency_ms
-
-        # Model and provider tracking
-        if record.model_name:
-            aggregates.models_used.add(record.model_name)
-        if record.provider:
-            aggregates.providers_used.add(record.provider)
-
-        # Tool calls
-        aggregates.tool_calls_count += len(record.output_tool_calls)
-
-        # Success/error tracking
-        if record.outcome == "error":
-            aggregates.error_count += 1
-        elif record.outcome == "success":
-            aggregates.success_count += 1
-
-    summary: AggregateSummary = {
-        "call_count": aggregates.call_count,
-        "input_tokens": aggregates.input_tokens,
-        "output_tokens": aggregates.output_tokens,
-        "total_tokens": aggregates.total_tokens,
-        "reasoning_tokens": aggregates.reasoning_tokens,
-        "cost_usd": aggregates.cost_usd,
-        "latency_ms": aggregates.latency_ms,
-        "models_used": list(aggregates.models_used),
-        "providers_used": list(aggregates.providers_used),
-        "tool_calls_count": aggregates.tool_calls_count,
-        "error_count": aggregates.error_count,
-        "success_count": aggregates.success_count,
-    }
-
-    if aggregates.call_count > 0:
-        summary["avg_latency_ms"] = aggregates.latency_ms / aggregates.call_count
-        summary["avg_input_tokens"] = aggregates.input_tokens / aggregates.call_count
-        summary["avg_output_tokens"] = aggregates.output_tokens / aggregates.call_count
-
-    return summary
-
-
-def create_llm_call_record_from_streaming(
-    chunks: list[LLMChunk],
-    model_name: str,
-    provider: str,
-    messages: list[dict[str, Any]],
-    temperature: float = 0.8,
-    request_params: dict[str, Any] | None = None,
-    started_at: datetime | None = None,
-    completed_at: datetime | None = None,
-) -> LLMCallRecord:
-    """Create an LLMCallRecord from streaming chunks.
-
-    This function reconstructs a complete LLMCallRecord from streaming
-    response chunks, useful for Responses API or streaming Chat Completions.
-
-    Args:
-        chunks: List of LLMChunk instances from streaming
-        model_name: Name of the model used
-        provider: Provider name
-        messages: Input messages sent to the model
-        temperature: Temperature parameter used
-        request_params: Additional request parameters
-        started_at: When the request started
-        completed_at: When the request completed
-
-    Returns:
-        A populated LLMCallRecord instance
-    """
-    # Reconstruct output text from chunks
-    output_text = "".join(chunk.delta_text for chunk in chunks if chunk.delta_text)
-
-    # Calculate latency from chunk timestamps
-    latency_ms = None
-    if chunks and started_at:
-        last_chunk_time = chunks[-1].received_at
-        latency_ms = int((last_chunk_time - started_at).total_seconds() * 1000)
-
-    # Convert input messages
-    input_messages = []
-    for msg in messages:
-        role = msg.get("role", "user")
-        content = msg.get("content", "")
-
-        if isinstance(content, str):
-            parts = [LLMContentPart(type="text", text=content)]
-        else:
-            parts = [LLMContentPart(type="text", text=str(content))]
-
-        input_messages.append(LLMMessage(role=role, parts=parts))
-
-    # Create output message
-    output_messages = [
-        LLMMessage(role="assistant", parts=[LLMContentPart(type="text", text=output_text)])
-    ]
-
-    # Build request parameters
-    params = LLMRequestParams(temperature=temperature, raw_params=request_params or {})
-
-    # Create the call record
-    record = LLMCallRecord(
-        call_id=str(uuid.uuid4()),
-        api_type="responses",  # Streaming typically from Responses API
-        provider=provider,
-        model_name=model_name,
-        started_at=started_at or datetime.now(timezone.utc),
-        completed_at=completed_at or datetime.now(timezone.utc),
-        latency_ms=latency_ms,
-        request_params=params,
-        input_messages=input_messages,
-        output_messages=output_messages,
-        output_text=output_text,
-        chunks=chunks,
-        outcome="success",
-        metadata={"chunk_count": len(chunks), "streaming": True},
-    )
-
-    return record

synth_ai/tracing_v3/migration_helper.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python3
-"""
-Helper script to identify files that need migration from v2 to v3 tracing.
-"""
-
-import os
-import re
-
-
-def find_v2_imports(root_path: str = ".") -> list[tuple[str, list[str]]]:
-    """Find all Python files importing from tracing_v2."""
-    v2_files = []
-
-    # Patterns to search for
-    patterns = [
-        re.compile(r"from\s+synth_ai\.tracing_v2"),
-        re.compile(r"import\s+synth_ai\.tracing_v2"),
-        re.compile(r"from\s+\.\.tracing_v2"),  # Relative imports
-        re.compile(r"from\s+\.tracing_v2"),
-    ]
-
-    for root, dirs, files in os.walk(root_path):
-        # Skip v2 implementation itself
-        if "tracing_v2" in root:
-            continue
-
-        # Skip v3 implementation
-        if "tracing_v3" in root:
-            continue
-
-        # Skip .git and other hidden directories
-        dirs[:] = [d for d in dirs if not d.startswith(".")]
-
-        for file in files:
-            if file.endswith(".py"):
-                file_path = os.path.join(root, file)
-                try:
-                    with open(file_path) as f:
-                        content = f.read()
-
-                    matches = []
-                    for pattern in patterns:
-                        for match in pattern.finditer(content):
-                            matches.append(match.group(0))
-
-                    if matches:
-                        v2_files.append((file_path, matches))
-                except Exception as e:
-                    print(f"Error reading {file_path}: {e}")
-
-    return v2_files
-
-
-def categorize_files(v2_files: list[tuple[str, list[str]]]) -> dict:
-    """Categorize files by their type/location."""
-    categories = {"tests": [], "core_library": [], "examples": [], "debug_scripts": [], "other": []}
-
-    for file_path, imports in v2_files:
-        if "test" in file_path or "Test" in file_path:
-            categories["tests"].append((file_path, imports))
-        elif (
-            "debug" in file_path
-            or file_path.startswith("./test_")
-            or file_path.startswith("./isolate_")
-        ):
-            categories["debug_scripts"].append((file_path, imports))
-        elif "example" in file_path:
-            categories["examples"].append((file_path, imports))
-        elif any(
-            core in file_path
-            for core in ["synth_ai/lm/", "synth_ai/tui/", "synth_ai/environments/"]
-        ):
-            categories["core_library"].append((file_path, imports))
-        else:
-            categories["other"].append((file_path, imports))
-
-    return categories
-
-
-def print_migration_report():
-    """Print a report of files needing migration."""
-    print("=== Tracing v2 to v3 Migration Report ===\n")
-
-    v2_files = find_v2_imports()
-    categories = categorize_files(v2_files)
-
-    total_files = len(v2_files)
-    print(f"Total files using v2 tracing: {total_files}\n")
-
-    for category, files in categories.items():
-        if files:
-            print(f"\n{category.upper().replace('_', ' ')} ({len(files)} files):")
-            print("-" * 50)
-            for file_path, imports in sorted(files):
-                print(f"  {file_path}")
-                for imp in imports[:3]:  # Show first 3 imports
-                    print(f"    - {imp}")
-                if len(imports) > 3:
-                    print(f"    ... and {len(imports) - 3} more imports")
-
-    print("\n\nRECOMMENDATIONS:")
-    print("-" * 50)
-    print("1. Test files: These are already configured to be skipped in pytest.ini")
-    print("2. Debug scripts: Can be deleted or archived")
-    print("3. Core library files: Need careful migration to v3")
-    print("   - synth_ai/lm/core/main_v2.py")
-    print("   - synth_ai/tui/cli/query_experiments.py")
-    print("   - synth_ai/environments/service/core_routes.py")
-    print("4. Examples: Should be updated to demonstrate v3 usage")
-
-    print("\n\nNEXT STEPS:")
-    print("-" * 50)
-    print("1. Update core library files to support v3 tracing")
-    print("2. Create v3 versions of essential tests")
-    print("3. Archive or remove debug scripts")
-    print("4. Update documentation to reference v3")
-
-
-if __name__ == "__main__":
-    print_migration_report()