turingpulse-sdk-autogen 1.0.0__tar.gz

turingpulse_sdk_autogen-1.0.0/.gitignore

@@ -0,0 +1,42 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+
+# Database files
+*.db
+*.sqlite3
+
+# Environment variables
+.env
+.env.local
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Logs
+*.log
+logs/
+
+# OS files
+.DS_Store
+Thumbs.db

turingpulse_sdk_autogen-1.0.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: turingpulse-sdk-autogen
+Version: 1.0.0
+Summary: TuringPulse SDK integration for AutoGen
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: pyautogen>=0.7.5
+Requires-Dist: turingpulse-sdk>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'

turingpulse_sdk_autogen-1.0.0/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "turingpulse-sdk-autogen"
+version = "1.0.0"
+description = "TuringPulse SDK integration for AutoGen"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+dependencies = [
+    "turingpulse-sdk>=1.0.0",
+    "pyautogen>=0.7.5",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "pytest-asyncio>=0.23"]

turingpulse_sdk_autogen-1.0.0/turingpulse_sdk_autogen/__init__.py

@@ -0,0 +1,6 @@
+"""TuringPulse SDK integration for AutoGen."""
+
+from ._wrapper import instrument_autogen
+
+__version__ = "0.1.0"
+__all__ = ["instrument_autogen"]

turingpulse_sdk_autogen-1.0.0/turingpulse_sdk_autogen/_wrapper.py

@@ -0,0 +1,334 @@
+"""AutoGen 0.2.x (pyautogen) instrumentation for TuringPulse SDK.
+
+Wraps ``user_proxy.initiate_chat()`` to capture:
+
+- Aggregate token / cost from ``chat_result.cost`` (keyed by model).
+- Per-turn child spans by walking ``chat_result.chat_history``.
+- Tool-call spans when messages contain ``tool_calls`` or ``role == "tool"``.
+- LLM reasoning spans for assistant messages.
+
+**Known shortfalls (AutoGen 0.2.x)**
+
+1. ``chat_result.cost`` is a model-level aggregate — per-turn token
+   counts are **not available**. We distribute totals evenly across
+   LLM spans.
+2. Tool execution results in chat history don't include timing info.
+3. ``chat_result.summary`` is often a plain string, not a structured
+   dict — token extraction from summary is unreliable.
+
+Usage::
+
+    from turingpulse_sdk.integrations.autogen import instrument_autogen
+
+    run_workflow = instrument_autogen(
+        user_proxy, assistant,
+        name="my-autogen-workflow",
+    )
+    result = run_workflow(message="Handle this request", max_turns=10)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from turingpulse_sdk.config import MAX_FIELD_SIZE
+from turingpulse_sdk.context import current_context
+from turingpulse_sdk import instrument, GovernanceDirective
+from turingpulse_sdk.integrations.base import emit_child_spans
+
+logger = logging.getLogger("turingpulse.sdk.integrations.autogen")
+
+FRAMEWORK_NAME = "autogen"
+FRAMEWORK_VERSION = "0.2.35"
+
+
+def instrument_autogen(
+    user_proxy: Any,
+    assistant: Any,
+    *,
+    name: str,
+    governance: Optional[GovernanceDirective] = None,
+    model: str = "gpt-4o-mini",
+    provider: str = "openai",
+    tool_names: Optional[List[str]] = None,
+    kpis: Optional[Sequence["KPIConfig"]] = None,
+    metadata: Optional[Dict[str, str]] = None,
+) -> Callable[..., Any]:
+    """Instrument an AutoGen chat for TuringPulse observability.
+
+    Returns a callable that runs ``user_proxy.initiate_chat(assistant, …)``
+    and emits a root workflow span plus per-turn child spans.
+
+    Args:
+        user_proxy: The ``ConversableAgent`` acting as user proxy.
+        assistant: The ``ConversableAgent`` acting as the LLM assistant.
+        name: Workflow display name for TuringPulse.
+        governance: Optional governance directive for policy enforcement.
+        model: LLM model name (default ``gpt-4o-mini``).
+        provider: LLM provider (default ``openai``).
+        tool_names: Names of registered tools for metadata.
+
+    Returns:
+        A callable ``run(message=..., **chat_kwargs) -> ChatResult``.
+    """
+    system_msg = getattr(assistant, "system_message", "") or ""
+
+    @instrument(name=name, governance=governance, kpis=kpis, metadata=metadata or {})
+    def run(message: str, **chat_kwargs: Any) -> Any:
+        t0 = time.time()
+        chat_result = user_proxy.initiate_chat(
+            assistant,
+            message=message,
+            **chat_kwargs,
+        )
+        total_duration_ms = int((time.time() - t0) * 1000)
+
+        ctx = current_context()
+        if ctx is None:
+            logger.warning("instrument_autogen: no active context")
+            return chat_result
+
+        # ── Extract aggregate cost / tokens ──
+        total_prompt, total_completion, total_cost = _extract_cost(chat_result)
+
+        ctx.set_tokens(total_prompt, total_completion)
+        ctx.set_cost(total_cost)
+        ctx.set_model(model, provider)
+        ctx.set_prompt(message[:MAX_FIELD_SIZE], system_msg[:MAX_FIELD_SIZE])
+        ctx.framework = FRAMEWORK_NAME
+        ctx.node_type = "workflow"
+
+        # Final output — last non-empty assistant message
+        chat_history = getattr(chat_result, "chat_history", []) or []
+        final_output = _extract_final_output(chat_history)
+        ctx.set_io(input_data=message[:MAX_FIELD_SIZE], output_data=final_output[:MAX_FIELD_SIZE])
+
+        if tool_names:
+            ctx.available_tools = tool_names
+
+        # ── Build child spans from chat history ──
+        child_spans = _build_chat_spans(
+            chat_history, total_duration_ms,
+            total_prompt, total_completion, total_cost,
+            model, provider, message, system_msg,
+            tool_names or [],
+        )
+        emit_child_spans(
+            child_spans,
+            run_id=ctx.run_id,
+            parent_span_id=ctx.span_id,
+            workflow_name=name,
+            framework=FRAMEWORK_NAME,
+        )
+
+        return chat_result
+
+    return run
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────
+
+
+def _extract_cost(chat_result: Any) -> tuple[int, int, float]:
+    """Extract total tokens and cost from ``chat_result.cost``.
+
+    AutoGen 0.2.x ``ChatResult.cost`` has a nested structure::
+
+        {
+            "usage_including_cached_inference": {
+                "total_cost": 0.000548,
+                "gpt-4o-mini-2024-07-18": {
+                    "cost": 0.000548,
+                    "prompt_tokens": 2425,
+                    "completion_tokens": 308,
+                    "total_tokens": 2733,
+                }
+            },
+            "usage_excluding_cached_inference": { ... }
+        }
+
+    We prefer ``usage_including_cached_inference`` for the complete picture.
+    """
+    cost_info = getattr(chat_result, "cost", {}) or {}
+    total_prompt = 0
+    total_completion = 0
+    total_cost = 0.0
+
+    if not isinstance(cost_info, dict):
+        return total_prompt, total_completion, total_cost
+
+    # Prefer usage_including_cached_inference, fallback to any key
+    usage_data = cost_info.get("usage_including_cached_inference")
+    if not isinstance(usage_data, dict):
+        usage_data = cost_info.get("usage_excluding_cached_inference")
+    if not isinstance(usage_data, dict):
+        # Try treating cost_info itself as the usage map (flat format)
+        usage_data = cost_info
+
+    if isinstance(usage_data, dict):
+        for key, value in usage_data.items():
+            if isinstance(value, dict) and "prompt_tokens" in value:
+                total_prompt += value.get("prompt_tokens", 0)
+                total_completion += value.get("completion_tokens", 0)
+                total_cost += value.get("cost", 0.0)
+
+    return total_prompt, total_completion, total_cost
+
+
+def _extract_final_output(chat_history: List[Dict[str, Any]]) -> str:
+    """Get the last non-TERMINATE assistant message."""
+    for msg in reversed(chat_history):
+        content = msg.get("content", "") or ""
+        if content and "TERMINATE" not in content:
+            return content
+    for msg in reversed(chat_history):
+        content = msg.get("content", "") or ""
+        if content:
+            return content.replace("TERMINATE", "").strip()
+    return ""
+
+
+def _build_chat_spans(
+    chat_history: List[Dict[str, Any]],
+    total_duration_ms: int,
+    total_prompt: int,
+    total_completion: int,
+    total_cost: float,
+    model: str,
+    provider: str,
+    user_message: str,
+    system_msg: str,
+    tool_names: List[str],
+) -> List[Dict[str, Any]]:
+    """Walk AutoGen chat history and build child span dicts."""
+    spans: List[Dict[str, Any]] = []
+
+    num_messages = max(len(chat_history), 1)
+    llm_messages = [
+        m for m in chat_history
+        if m.get("tool_calls") or (
+            m.get("content") and m.get("role") not in ("user", "tool")
+        )
+    ]
+    num_llm = max(len(llm_messages), 1)
+    per_llm_prompt = total_prompt // num_llm
+    per_llm_completion = total_completion // num_llm
+    per_llm_cost = 0  # Backend calculates cost based on model pricing
+
+    llm_count = 0
+    tool_count = 0
+
+    # Two-pass approach: First pass collects tool calls, second pass links
+    # tool results back to the tool call records for proper success/error
+    # tracking (AUDIT-FIX for AutoGen two-pass linking).
+
+    # Pass 1: Build a mapping of tool_call_id -> tool result content from
+    # all tool-result messages that follow the tool call requests.
+    tool_result_map: dict = {}
+    for msg in chat_history:
+        if msg.get("role") == "tool":
+            tc_id = msg.get("tool_call_id", "")
+            if tc_id:
+                tool_result_map[tc_id] = msg.get("content", "") or ""
+
+    # Pass 2: Process messages and create spans with linked tool results
+    for i, msg in enumerate(chat_history):
+        role = msg.get("role", msg.get("name", "unknown"))
+        content = msg.get("content", "") or ""
+        tool_calls = msg.get("tool_calls", [])
+
+        if tool_calls:
+            llm_count += 1
+            tc_data = []
+            for tc in tool_calls:
+                fn = tc.get("function", {})
+                args_raw = fn.get("arguments", "{}")
+                try:
+                    args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw or {})
+                except (json.JSONDecodeError, TypeError):
+                    args = {"raw": str(args_raw)[:500]}
+                tc_id = tc.get("id", "")
+
+                # Link tool result from the result map
+                linked_result = tool_result_map.get(tc_id)
+                has_result = linked_result is not None
+                tc_success = has_result
+                tc_error = None if has_result else "No tool result found in chat history"
+
+                tc_data.append({
+                    "tool_name": fn.get("name", "unknown"),
+                    "tool_args": args,
+                    "tool_result": str(linked_result)[:MAX_FIELD_SIZE] if linked_result else "",
+                    "tool_id": tc_id,
+                    "success": tc_success,
+                    "error_message": tc_error,
+                })
+
+            spans.append({
+                "node": f"llm_reasoning_{llm_count}",
+                "node_type": "llm",
+                "duration_ms": total_duration_ms // num_messages,
+                "status": "success",
+                "prompt": user_message[:MAX_FIELD_SIZE],
+                "system_prompt": system_msg[:MAX_FIELD_SIZE],
+                "input": {"message_index": i, "role": role},
+                "output": {"content": content[:MAX_FIELD_SIZE], "tool_calls_count": len(tool_calls)},
+                "model": model,
+                "provider": provider,
+                "tokens": {"prompt": per_llm_prompt, "completion": per_llm_completion},
+                "cost_usd": round(per_llm_cost, 6),
+                "tool_calls": tc_data,
+                "available_tools": tool_names,
+                "metadata": {
+                    "autogen_agent": str(role),
+                    "autogen_message_type": "assistant_with_tools",
+                    "autogen_conversation_turn": str(i),
+                    "autogen_version": FRAMEWORK_VERSION,
+                },
+            })
+
+        elif msg.get("role") == "tool":
+            tool_count += 1
+            tool_name = msg.get("name", f"tool_{tool_count}")
+            spans.append({
+                "node": f"tool_{tool_name}",
+                "node_type": "tool",
+                "duration_ms": total_duration_ms // num_messages,
+                "status": "success",
+                "input": {"tool_call_id": msg.get("tool_call_id", "")},
+                "output": {"result": content[:MAX_FIELD_SIZE]},
+                "available_tools": tool_names,
+                "metadata": {
+                    "autogen_tool_name": tool_name,
+                    "autogen_message_type": "tool_result",
+                    "autogen_version": FRAMEWORK_VERSION,
+                },
+            })
+
+        elif content and role != "user":
+            llm_count += 1
+            spans.append({
+                "node": f"assistant_response_{llm_count}",
+                "node_type": "llm",
+                "duration_ms": total_duration_ms // num_messages,
+                "status": "success",
+                "prompt": user_message[:MAX_FIELD_SIZE],
+                "system_prompt": system_msg[:MAX_FIELD_SIZE],
+                "input": {"conversation_context": f"Turn {i} in AutoGen chat"},
+                "output": {"response": content[:MAX_FIELD_SIZE]},
+                "model": model,
+                "provider": provider,
+                "tokens": {"prompt": per_llm_prompt, "completion": per_llm_completion},
+                "cost_usd": round(per_llm_cost, 6),
+                "metadata": {
+                    "autogen_agent": str(role),
+                    "autogen_message_type": "assistant_response",
+                    "autogen_conversation_turn": str(i),
+                    "autogen_version": FRAMEWORK_VERSION,
+                },
+            })
+
+    return spans