auditi 0.1.0__tar.gz → 0.1.1__tar.gz

{auditi-0.1.0 → auditi-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: auditi
-Version: 0.1.0
+Version: 0.1.1
 Summary: Trace, monitor, and evaluate AI agents and LLM applications with simple decorators
 Project-URL: Repository, https://github.com/deduu/auditi
 Project-URL: Issues, https://github.com/deduu/auditi/issues

{auditi-0.1.0 → auditi-0.1.1}/auditi/__init__.py

@@ -15,6 +15,8 @@ from .transport import BaseTransport, SyncHttpTransport, DebugTransport
 from .types import TraceInput, SpanInput, EvaluationResult
 from .events import EventType, StreamEvent
 
+from .instrumentation import instrument
+
 __version__ = "0.1.0"
 
 __all__ = [
@@ -22,6 +24,8 @@ __all__ = [
     "init",
     "get_client",
     "AuditiClient",
+    # Instrumentation
+    "instrument",
     # Context
     "set_context",
     "get_context",

{auditi-0.1.0 → auditi-0.1.1}/auditi/decorators.py

@@ -127,6 +127,33 @@ def _apply_usage_to_trace(trace: TraceInput, usage: Any, model: Optional[str] =
     trace.cost = (trace.cost or 0.0) + incremental_cost
 
 
+def _ensure_str_input(value: Any) -> str:
+    """
+    Convert various input formats to a plain string for user_input.
+
+    Handles:
+      - str: returned as-is
+      - list of message dicts (OpenAI/chat format): extracts last user message content
+      - dict with 'content' or 'message' key: extracts the value
+      - anything else: str() conversion
+    """
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        # Try to extract the last user message from chat-format messages
+        for msg in reversed(value):
+            if isinstance(msg, dict) and msg.get("role") == "user":
+                content = msg.get("content", "")
+                return str(content) if not isinstance(content, str) else content
+        # Fallback: stringify the list
+        return str(value)
+    if isinstance(value, dict):
+        return value.get("content") or value.get("message") or str(value)
+    if value is None:
+        return ""
+    return str(value)
+
+
 def _execute_as_standalone_trace(
     func: Callable,
     args: tuple,
@@ -151,19 +178,11 @@ def _execute_as_standalone_trace(
     # Extract user input from args/kwargs
     user_input = ""
     if args:
-        first_arg = args[0]
-        if isinstance(first_arg, str):
-            user_input = first_arg
-        elif isinstance(first_arg, list):
-            # Could be messages array
-            user_input = str(first_arg)
-        elif isinstance(first_arg, dict):
-            user_input = first_arg.get("content") or first_arg.get("message") or str(first_arg)
-        else:
-            user_input = str(first_arg)
+        user_input = _ensure_str_input(args[0])
 
     if not user_input:
-        user_input = kwargs.get("prompt") or kwargs.get("message") or kwargs.get("query") or ""
+        raw = kwargs.get("prompt") or kwargs.get("message") or kwargs.get("messages") or kwargs.get("query") or ""
+        user_input = _ensure_str_input(raw)
 
     # Create trace
     trace = TraceInput(
@@ -352,25 +371,19 @@ def trace_agent(
 
                     # Get the actual user input from the correct position
                     if len(args) > start_idx:
-                        first_arg = args[start_idx]
-                        if isinstance(first_arg, str):
-                            user_input = first_arg
-                        elif isinstance(first_arg, dict) and "message" in first_arg:
-                            user_input = first_arg["message"]
-                        elif isinstance(first_arg, dict) and "content" in first_arg:
-                            user_input = first_arg["content"]
-                        else:
-                            user_input = str(first_arg)
+                        user_input = _ensure_str_input(args[start_idx])
 
                 # Also check for user_input/message/query in kwargs
                 if not user_input:
-                    user_input = (
+                    raw = (
                         kwargs.get("user_input")
                         or kwargs.get("message")
+                        or kwargs.get("messages")
                         or kwargs.get("query")
                         or kwargs.get("prompt")
                         or ""
                     )
+                    user_input = _ensure_str_input(raw)
 
                 _debug_log(f"Captured user input:", {"user_input": user_input[:200]})
 
@@ -544,25 +557,19 @@ def trace_agent(
 
                     # Get the actual user input from the correct position
                     if len(args) > start_idx:
-                        first_arg = args[start_idx]
-                        if isinstance(first_arg, str):
-                            user_input = first_arg
-                        elif isinstance(first_arg, dict) and "message" in first_arg:
-                            user_input = first_arg["message"]
-                        elif isinstance(first_arg, dict) and "content" in first_arg:
-                            user_input = first_arg["content"]
-                        else:
-                            user_input = str(first_arg)
+                        user_input = _ensure_str_input(args[start_idx])
 
                 # Also check for user_input/message/query in kwargs
                 if not user_input:
-                    user_input = (
+                    raw = (
                         kwargs.get("user_input")
                         or kwargs.get("message")
+                        or kwargs.get("messages")
                         or kwargs.get("query")
                         or kwargs.get("prompt")
                         or ""
                     )
+                    user_input = _ensure_str_input(raw)
 
                 _debug_log(f"Captured user input:", {"user_input": user_input[:200]})
 
@@ -747,25 +754,19 @@ def trace_agent(
 
                     # Get the actual user input from the correct position
                     if len(args) > start_idx:
-                        first_arg = args[start_idx]
-                        if isinstance(first_arg, str):
-                            user_input = first_arg
-                        elif isinstance(first_arg, dict) and "message" in first_arg:
-                            user_input = first_arg["message"]
-                        elif isinstance(first_arg, dict) and "content" in first_arg:
-                            user_input = first_arg["content"]
-                        else:
-                            user_input = str(first_arg)
+                        user_input = _ensure_str_input(args[start_idx])
 
                 # Also check for user_input/message/query in kwargs
                 if not user_input:
-                    user_input = (
+                    raw = (
                         kwargs.get("user_input")
                         or kwargs.get("message")
+                        or kwargs.get("messages")
                         or kwargs.get("query")
                         or kwargs.get("prompt")
                         or ""
                     )
+                    user_input = _ensure_str_input(raw)
 
                 _debug_log(f"Captured user input:", {"user_input": user_input[:200]})
 

auditi-0.1.1/auditi/instrumentation.py

@@ -0,0 +1,105 @@
+import importlib
+import functools
+from typing import Any, Callable, Optional
+import logging
+
+from .decorators import trace_llm
+from .client import get_client
+
+logger = logging.getLogger("auditi.instrumentation")
+
+
+def instrument(
+    openai: bool = True,
+    anthropic: bool = True,
+    langchain: bool = False,  # Placeholder for future
+):
+    """
+    Automatically instrument installed libraries to capture traces.
+
+    Args:
+        openai: Whether to instrument the OpenAI library
+        anthropic: Whether to instrument the Anthropic library
+        langchain: Whether to instrument LangChain (not yet implemented)
+    """
+    if openai:
+        _instrument_openai()
+
+    if anthropic:
+        _instrument_anthropic()
+
+
+def _instrument_openai():
+    """Patch OpenAI client methods"""
+    try:
+        import openai
+        from openai import OpenAI, AsyncOpenAI
+        from openai.resources.chat.completions import Completions, AsyncCompletions
+    except ImportError:
+        logger.debug("OpenAI library not found, skipping instrumentation")
+        return
+
+    logger.info("Instrumenting OpenAI...")
+
+    # We patch the 'create' method of the Completions resource
+    # checks if already patched to avoid double patching
+    if getattr(Completions.create, "_is_auditi_patched", False):
+        return
+
+    original_create = Completions.create
+    original_async_create = AsyncCompletions.create
+
+    @trace_llm(name="OpenAI Chat Completion", standalone=True)
+    def patched_create(self, *args, **kwargs):
+        # We need to make sure we don't double-trace if the user is already using decorators
+        # But for now, simple wrapping using our existing @trace_llm is easiest.
+        # It handles standalone vs nested automatically.
+        return original_create(self, *args, **kwargs)
+
+    @trace_llm(name="OpenAI Chat Completion", standalone=True)
+    async def patched_async_create(self, *args, **kwargs):
+        return await original_async_create(self, *args, **kwargs)
+
+    # Mark as patched
+    patched_create._is_auditi_patched = True
+    patched_async_create._is_auditi_patched = True
+
+    # Apply patches
+    Completions.create = patched_create
+    AsyncCompletions.create = patched_async_create
+
+    logger.info("OpenAI instrumentation applied successfully")
+
+
+def _instrument_anthropic():
+    """Patch Anthropic client methods"""
+    try:
+        import anthropic
+        from anthropic.resources.messages import Messages, AsyncMessages
+    except ImportError:
+        logger.debug("Anthropic library not found, skipping instrumentation")
+        return
+
+    logger.info("Instrumenting Anthropic...")
+
+    if getattr(Messages.create, "_is_auditi_patched", False):
+        return
+
+    original_create = Messages.create
+    original_async_create = AsyncMessages.create
+
+    @trace_llm(name="Anthropic Message", standalone=True)
+    def patched_create(self, *args, **kwargs):
+        return original_create(self, *args, **kwargs)
+
+    @trace_llm(name="Anthropic Message", standalone=True)
+    async def patched_async_create(self, *args, **kwargs):
+        return await original_async_create(self, *args, **kwargs)
+
+    patched_create._is_auditi_patched = True
+    patched_async_create._is_auditi_patched = True
+
+    Messages.create = patched_create
+    AsyncMessages.create = patched_async_create
+
+    logger.info("Anthropic instrumentation applied successfully")

{auditi-0.1.0 → auditi-0.1.1}/auditi/types/api_types.py

@@ -82,7 +82,18 @@ class TraceInput(BaseModel):
 
     @field_validator("user_input", mode="before")
     def normalize_user_input(cls, v):
-        return v or ""
+        if isinstance(v, str):
+            return v
+        if isinstance(v, list):
+            # Extract last user message from chat-format messages
+            for msg in reversed(v):
+                if isinstance(msg, dict) and msg.get("role") == "user":
+                    content = msg.get("content", "")
+                    return str(content) if not isinstance(content, str) else content
+            return str(v)
+        if v is None:
+            return ""
+        return str(v)
 
 
 class EvaluationResult(BaseModel):

{auditi-0.1.0 → auditi-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "auditi"
-version = "0.1.0"
+version = "0.1.1"
 description = "Trace, monitor, and evaluate AI agents and LLM applications with simple decorators"
 readme = "README.md"
 license = {text = "MIT"}

{auditi-0.1.0 → auditi-0.1.1}/tests/test_decorators.py

@@ -1,7 +1,7 @@
 import pytest
 from unittest.mock import MagicMock, patch
 import asyncio
-from auditi.decorators import trace_agent, trace_tool, trace_llm
+from auditi.decorators import trace_agent, trace_tool, trace_llm, _ensure_str_input
 from auditi.client import init
 from auditi.context import clear_current_trace
 from auditi.types import TraceInput
@@ -102,3 +102,64 @@ def test_trace_llm_standalone(mock_client):
     # Based on my read, it calls _execute_as_standalone_trace if no current trace.
 
     assert mock_client.transport.send_trace.called
+
+
+# --- Tests for _ensure_str_input and message-list handling ---
+
+
+def test_ensure_str_input_with_string():
+    assert _ensure_str_input("hello") == "hello"
+
+
+def test_ensure_str_input_with_none():
+    assert _ensure_str_input(None) == ""
+
+
+def test_ensure_str_input_with_message_list():
+    messages = [
+        {"role": "user", "content": "What is AI?"},
+        {"role": "assistant", "content": "AI is..."},
+        {"role": "user", "content": "Tell me more"},
+    ]
+    assert _ensure_str_input(messages) == "Tell me more"
+
+
+def test_ensure_str_input_with_single_user_message():
+    messages = [{"role": "user", "content": "Hello world"}]
+    assert _ensure_str_input(messages) == "Hello world"
+
+
+def test_ensure_str_input_with_dict():
+    assert _ensure_str_input({"content": "test"}) == "test"
+    assert _ensure_str_input({"message": "test2"}) == "test2"
+
+
+def test_trace_agent_with_messages_list(mock_client):
+    """Test that trace_agent handles OpenAI-format message lists without crashing."""
+
+    @trace_agent(name="ChatAgent")
+    def chat_agent(messages):
+        return f"Response to: {messages[-1]['content']}"
+
+    messages = [{"role": "user", "content": "Jelaskan WAP"}]
+    result = chat_agent(messages)
+
+    assert result == "Response to: Jelaskan WAP"
+    mock_client.transport.send_trace.assert_called_once()
+
+    call_args = mock_client.transport.send_trace.call_args[0][0]
+    assert call_args["user_input"] == "Jelaskan WAP"
+
+
+def test_trace_input_validator_with_list():
+    """Test that TraceInput.normalize_user_input handles list input."""
+    from uuid import uuid4
+    from datetime import datetime
+
+    trace = TraceInput(
+        id=uuid4(),
+        start_time=datetime.utcnow(),
+        name="test",
+        user_input=[{"role": "user", "content": "Hello from list"}],
+    )
+    assert trace.user_input == "Hello from list"

auditi-0.1.1/tests/test_instrumentation.py

@@ -0,0 +1,74 @@
+import pytest
+from unittest.mock import MagicMock, patch
+import auditi
+from auditi.instrumentation import instrument
+
+
+class MockOpenAIHeader:
+    def __init__(self, content):
+        self.content = content
+
+
+class MockOpenAIChoice:
+    def __init__(self, content):
+        self.message = MockOpenAIHeader(content)
+
+
+class MockOpenAIResponse:
+    def __init__(self, content):
+        self.choices = [MockOpenAIChoice(content)]
+        self.model = "gpt-4"
+        self.usage = {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}
+
+
+def test_instrumentation_patches_openai():
+    # Setup mock OpenAI
+    mock_create = MagicMock(return_value=MockOpenAIResponse("Hello world"))
+
+    with (
+        patch("auditi.instrumentation.Completions") as MockCompletions,
+        patch("auditi.instrumentation.AsyncCompletions") as MockAsyncCompletions,
+    ):
+
+        # Setup the mock to be patched
+        MockCompletions.create = mock_create
+
+        # Run instrumentation
+        instrument(openai=True, anthropic=False)
+
+        # Verify it was patched (the new create should be a function, likely wrapped)
+        assert MockCompletions.create != mock_create
+        assert getattr(MockCompletions.create, "_is_auditi_patched", False) == False
+        # Wait, my implementation sets the attribute on the wrapper function *before* assigning it to the class method
+        # But when we access it via Class.method, it might be unbound or bound depending on Python version/mock.
+        # However, checking if it's different from original is a good start.
+
+        # Let's verify that calling the patched method calls the original method
+        # We need to mock the trace_llm decorator to avoid actual key/http logic
+
+        # Actually, simpler test: call it and see if trace_llm logic runs.
+        # But trace_llm needs an initialized client.
+
+        # Setup auditi client mock
+        mock_client = MagicMock()
+        with patch("auditi.decorators.get_client", return_value=mock_client):
+            # Call the patched method
+            # Note: The patched method expects 'self' as first arg because it's replacing a method on a resource class
+            # But usually SDKs use instances.
+            # `Completions.create` in OpenAI v1 is an instance method.
+            # My instrumentation patches the class method `Completions.create`.
+
+            # Create a dummy self
+            dummy_self = MagicMock()
+
+            # Call it!
+            response = MockCompletions.create(dummy_self, model="gpt-4", messages=[])
+
+            # Verify result passed through
+            assert response.choices[0].message.content == "Hello world"
+
+            # Verify client trace started (send_trace called)
+            mock_client.transport.send_trace.assert_called()
+
+            # Verify original create called
+            mock_create.assert_called()