agno 2.3.4__py3-none-any.whl → 2.3.6__py3-none-any.whl

agno/tracing/__init__.py

@@ -0,0 +1,12 @@
+"""
+Agno Tracing Module
+
+This module provides OpenTelemetry-based tracing capabilities for Agno agents.
+It uses the openinference-instrumentation-agno package for automatic instrumentation
+and provides a custom DatabaseSpanExporter to store traces in the Agno database.
+"""
+
+from agno.tracing.exporter import DatabaseSpanExporter
+from agno.tracing.setup import setup_tracing
+
+__all__ = ["DatabaseSpanExporter", "setup_tracing"]

agno/tracing/exporter.py

@@ -0,0 +1,157 @@
+"""
+Custom OpenTelemetry SpanExporter that writes traces to Agno database.
+"""
+
+import asyncio
+from collections import defaultdict
+from typing import Dict, List, Sequence, Union
+
+from opentelemetry.sdk.trace import ReadableSpan  # type: ignore
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult  # type: ignore
+
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.tracing.schemas import Span, create_trace_from_spans
+from agno.utils.log import logger
+
+
+class DatabaseSpanExporter(SpanExporter):
+    """Custom OpenTelemetry SpanExporter that writes to Agno database"""
+
+    def __init__(self, db: Union[BaseDb, AsyncBaseDb]):
+        """
+        Initialize the DatabaseSpanExporter.
+
+        Args:
+            db: Database instance (sync or async) to store traces
+        """
+        self.db = db
+        self._shutdown = False
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        """
+        Export spans to the database.
+
+        This method:
+        1. Converts OpenTelemetry spans to Span objects
+        2. Groups spans by trace_id
+        3. Creates Trace records (one per trace_id)
+        4. Creates Span records (multiple per trace_id)
+
+        Args:
+            spans: Sequence of OpenTelemetry ReadableSpan objects
+
+        Returns:
+            SpanExportResult indicating success or failure
+        """
+        if self._shutdown:
+            logger.warning("DatabaseSpanExporter is shutdown, cannot export spans")
+            return SpanExportResult.FAILURE
+
+        if not spans:
+            return SpanExportResult.SUCCESS
+
+        try:
+            # Convert OpenTelemetry spans to Span objects
+            converted_spans: List[Span] = []
+            for span in spans:
+                try:
+                    converted_span = Span.from_otel_span(span)
+                    converted_spans.append(converted_span)
+                except Exception as e:
+                    logger.error(f"Failed to convert span {span.name}: {e}")
+                    # Continue processing other spans
+                    continue
+
+            if not converted_spans:
+                return SpanExportResult.SUCCESS
+
+            # Group spans by trace_id
+            spans_by_trace: Dict[str, List[Span]] = defaultdict(list)
+            for converted_span in converted_spans:
+                spans_by_trace[converted_span.trace_id].append(converted_span)
+
+            # Handle async DB
+            if isinstance(self.db, AsyncBaseDb):
+                self._export_async(spans_by_trace)
+            else:
+                # Synchronous database
+                self._export_sync(spans_by_trace)
+
+            return SpanExportResult.SUCCESS
+        except Exception as e:
+            logger.error(f"Failed to export spans to database: {e}", exc_info=True)
+            return SpanExportResult.FAILURE
+
+    def _export_sync(self, spans_by_trace: Dict[str, List[Span]]) -> None:
+        """Export traces and spans to synchronous database"""
+        try:
+            # Create trace and span records for each trace
+            for trace_id, spans in spans_by_trace.items():
+                # Create trace record (aggregate of all spans)
+                trace = create_trace_from_spans(spans)
+                if trace:
+                    self.db.create_trace(trace)
+
+                # Create span records
+                self.db.create_spans(spans)
+
+        except Exception as e:
+            logger.error(f"Failed to export sync traces: {e}", exc_info=True)
+            raise
+
+    def _export_async(self, spans_by_trace: Dict[str, List[Span]]) -> None:
+        """Handle async database export"""
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                # We're in an async context, schedule the coroutine
+                asyncio.create_task(self._do_async_export(spans_by_trace))
+            else:
+                # No running loop, run in new loop
+                loop.run_until_complete(self._do_async_export(spans_by_trace))
+        except RuntimeError:
+            # No event loop, create new one
+            try:
+                asyncio.run(self._do_async_export(spans_by_trace))
+            except Exception as e:
+                logger.error(f"Failed to export async traces: {e}", exc_info=True)
+
+    async def _do_async_export(self, spans_by_trace: Dict[str, List[Span]]) -> None:
+        """Actually perform the async export"""
+        try:
+            # Create trace and span records for each trace
+            for trace_id, spans in spans_by_trace.items():
+                # Create trace record (aggregate of all spans)
+                trace = create_trace_from_spans(spans)
+                if trace:
+                    create_trace_result = self.db.create_trace(trace)
+                    if create_trace_result is not None:
+                        await create_trace_result
+
+                # Create span records
+                create_spans_result = self.db.create_spans(spans)
+                if create_spans_result is not None:
+                    await create_spans_result
+
+        except Exception as e:
+            logger.error(f"Failed to do async export: {e}", exc_info=True)
+            raise
+
+    def shutdown(self) -> None:
+        """Shutdown the exporter"""
+        self._shutdown = True
+        logger.debug("DatabaseSpanExporter shutdown")
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """
+        Force flush any pending spans.
+
+        Since we write immediately to the database, this is a no-op.
+
+        Args:
+            timeout_millis: Timeout in milliseconds
+
+        Returns:
+            True if flush was successful
+        """
+        return True

agno/tracing/schemas.py

@@ -0,0 +1,276 @@
+"""
+Trace data models for Agno tracing.
+"""
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from opentelemetry.sdk.trace import ReadableSpan  # type: ignore
+from opentelemetry.trace import SpanKind, StatusCode  # type: ignore
+
+
+@dataclass
+class Trace:
+    """Represents a complete trace (one record per trace_id)"""
+
+    trace_id: str
+    name: str  # Name from root span
+    status: str  # Overall status: OK, ERROR, UNSET
+    start_time: datetime  # Python datetime object
+    end_time: datetime  # Python datetime object
+    duration_ms: int
+    total_spans: int
+    error_count: int
+
+    # Context from root span
+    run_id: Optional[str]
+    session_id: Optional[str]
+    user_id: Optional[str]
+    agent_id: Optional[str]
+    team_id: Optional[str]
+    workflow_id: Optional[str]
+
+    created_at: datetime  # Python datetime object
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Trace to dictionary for database storage (datetime -> ISO string)"""
+        data = asdict(self)
+        # Convert datetime objects to ISO format strings for database storage
+        data["start_time"] = self.start_time.isoformat()
+        data["end_time"] = self.end_time.isoformat()
+        data["created_at"] = self.created_at.isoformat()
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Trace":
+        """Create Trace from dictionary (ISO string -> datetime)"""
+        # Convert ISO format strings to datetime objects
+        start_time = data["start_time"]
+        if isinstance(start_time, str):
+            start_time = datetime.fromisoformat(start_time.replace("Z", "+00:00"))
+        elif isinstance(start_time, int):
+            start_time = datetime.fromtimestamp(start_time / 1_000_000_000, tz=timezone.utc)
+
+        end_time = data["end_time"]
+        if isinstance(end_time, str):
+            end_time = datetime.fromisoformat(end_time.replace("Z", "+00:00"))
+        elif isinstance(end_time, int):
+            end_time = datetime.fromtimestamp(end_time / 1_000_000_000, tz=timezone.utc)
+
+        created_at = data["created_at"]
+        if isinstance(created_at, str):
+            created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+        elif isinstance(created_at, int):
+            created_at = datetime.fromtimestamp(created_at, tz=timezone.utc)
+
+        return cls(
+            trace_id=data["trace_id"],
+            name=data["name"],
+            status=data["status"],
+            start_time=start_time,
+            end_time=end_time,
+            duration_ms=data["duration_ms"],
+            total_spans=data["total_spans"],
+            error_count=data["error_count"],
+            run_id=data.get("run_id"),
+            session_id=data.get("session_id"),
+            user_id=data.get("user_id"),
+            agent_id=data.get("agent_id"),
+            team_id=data.get("team_id"),
+            workflow_id=data.get("workflow_id"),
+            created_at=created_at,
+        )
+
+
+@dataclass
+class Span:
+    """Represents a single span within a trace"""
+
+    span_id: str
+    trace_id: str
+    parent_span_id: Optional[str]
+    name: str
+    span_kind: str
+    status_code: str
+    status_message: Optional[str]
+    start_time: datetime  # Python datetime object
+    end_time: datetime  # Python datetime object
+    duration_ms: int
+    attributes: Dict[str, Any]
+    created_at: datetime  # Python datetime object
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert Span to dictionary for database storage (datetime -> ISO string)"""
+        data = asdict(self)
+        # Convert datetime objects to ISO format strings for database storage
+        data["start_time"] = self.start_time.isoformat()
+        data["end_time"] = self.end_time.isoformat()
+        data["created_at"] = self.created_at.isoformat()
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Span":
+        """Create Span from dictionary (ISO string -> datetime)"""
+        # Convert ISO format strings to datetime objects
+        start_time = data["start_time"]
+        if isinstance(start_time, str):
+            start_time = datetime.fromisoformat(start_time.replace("Z", "+00:00"))
+        elif isinstance(start_time, int):
+            start_time = datetime.fromtimestamp(start_time / 1_000_000_000, tz=timezone.utc)
+
+        end_time = data["end_time"]
+        if isinstance(end_time, str):
+            end_time = datetime.fromisoformat(end_time.replace("Z", "+00:00"))
+        elif isinstance(end_time, int):
+            end_time = datetime.fromtimestamp(end_time / 1_000_000_000, tz=timezone.utc)
+
+        created_at = data["created_at"]
+        if isinstance(created_at, str):
+            created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+        elif isinstance(created_at, int):
+            created_at = datetime.fromtimestamp(created_at, tz=timezone.utc)
+
+        return cls(
+            span_id=data["span_id"],
+            trace_id=data["trace_id"],
+            parent_span_id=data.get("parent_span_id"),
+            name=data["name"],
+            span_kind=data["span_kind"],
+            status_code=data["status_code"],
+            status_message=data.get("status_message"),
+            start_time=start_time,
+            end_time=end_time,
+            duration_ms=data["duration_ms"],
+            attributes=data.get("attributes", {}),
+            created_at=created_at,
+        )
+
+    @classmethod
+    def from_otel_span(cls, otel_span: ReadableSpan) -> "Span":
+        """Convert OpenTelemetry ReadableSpan to Span"""
+        # Extract span context
+        span_context = otel_span.context
+        trace_id = format(span_context.trace_id, "032x") if span_context else "0" * 32
+        span_id = format(span_context.span_id, "016x") if span_context else "0" * 16
+
+        # Extract parent span ID if exists
+        parent_span_id = None
+        if otel_span.parent and otel_span.parent.span_id:
+            parent_span_id = format(otel_span.parent.span_id, "016x")
+
+        # Extract span kind
+        span_kind_map = {
+            SpanKind.INTERNAL: "INTERNAL",
+            SpanKind.SERVER: "SERVER",
+            SpanKind.CLIENT: "CLIENT",
+            SpanKind.PRODUCER: "PRODUCER",
+            SpanKind.CONSUMER: "CONSUMER",
+        }
+        span_kind = span_kind_map.get(otel_span.kind, "INTERNAL")
+
+        # Extract status
+        status_code_map = {
+            StatusCode.UNSET: "UNSET",
+            StatusCode.OK: "OK",
+            StatusCode.ERROR: "ERROR",
+        }
+        status_code = status_code_map.get(otel_span.status.status_code, "UNSET")
+        status_message = otel_span.status.description
+
+        # Calculate duration in milliseconds
+        start_time_ns = otel_span.start_time or 0
+        end_time_ns = otel_span.end_time or start_time_ns
+        duration_ms = int((end_time_ns - start_time_ns) / 1_000_000)
+
+        # Convert nanosecond timestamps to datetime objects
+        start_time = datetime.fromtimestamp(start_time_ns / 1_000_000_000, tz=timezone.utc)
+        end_time = datetime.fromtimestamp(end_time_ns / 1_000_000_000, tz=timezone.utc)
+
+        # Convert attributes to dictionary
+        attributes: Dict[str, Any] = {}
+        if otel_span.attributes:
+            for key, value in otel_span.attributes.items():
+                # Convert attribute values to JSON-serializable types
+                if isinstance(value, (str, int, float, bool, type(None))):
+                    attributes[key] = value
+                elif isinstance(value, (list, tuple)):
+                    attributes[key] = list(value)
+                else:
+                    attributes[key] = str(value)
+
+        return cls(
+            span_id=span_id,
+            trace_id=trace_id,
+            parent_span_id=parent_span_id,
+            name=otel_span.name,
+            span_kind=span_kind,
+            status_code=status_code,
+            status_message=status_message,
+            start_time=start_time,
+            end_time=end_time,
+            duration_ms=duration_ms,
+            attributes=attributes,
+            created_at=datetime.now(timezone.utc),
+        )
+
+
+def create_trace_from_spans(spans: List[Span]) -> Optional[Trace]:
+    """
+    Create a Trace object from a list of Span objects with the same trace_id.
+
+    Args:
+        spans: List of Span objects belonging to the same trace
+
+    Returns:
+        Trace object with aggregated information, or None if spans list is empty
+    """
+    if not spans:
+        return None
+
+    # Find root span (no parent)
+    root_span = next((s for s in spans if not s.parent_span_id), spans[0])
+
+    # Calculate aggregated metrics
+    trace_id = spans[0].trace_id
+    start_time = min(s.start_time for s in spans)
+    end_time = max(s.end_time for s in spans)
+    duration_ms = int((end_time - start_time).total_seconds() * 1000)
+    total_spans = len(spans)
+    error_count = sum(1 for s in spans if s.status_code == "ERROR")
+
+    # Determine overall status (ERROR if any span errored, OK otherwise)
+    status = "ERROR" if error_count > 0 else "OK"
+
+    # Extract context from root span's attributes
+    attrs = root_span.attributes
+    run_id = attrs.get("run_id") or attrs.get("agno.run.id")
+
+    session_id = attrs.get("session_id") or attrs.get("agno.session.id") or attrs.get("session.id")
+
+    user_id = attrs.get("user_id") or attrs.get("agno.user.id") or attrs.get("user.id")
+
+    # Try to extract agent_id from the span name or attributes
+    agent_id = attrs.get("agent_id") or attrs.get("agno.agent.id")
+
+    team_id = attrs.get("team_id") or attrs.get("agno.team.id")
+
+    workflow_id = attrs.get("workflow_id") or attrs.get("agno.workflow.id")
+
+    return Trace(
+        trace_id=trace_id,
+        name=root_span.name,
+        status=status,
+        start_time=start_time,
+        end_time=end_time,
+        duration_ms=duration_ms,
+        total_spans=total_spans,
+        error_count=error_count,
+        run_id=run_id,
+        session_id=session_id,
+        user_id=user_id,
+        agent_id=agent_id,
+        team_id=team_id,
+        workflow_id=workflow_id,
+        created_at=datetime.now(timezone.utc),
+    )

agno/tracing/setup.py

@@ -0,0 +1,111 @@
+"""
+Setup helper functions for configuring Agno tracing.
+"""
+
+from typing import Union
+
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.tracing.exporter import DatabaseSpanExporter
+from agno.utils.log import logger
+
+try:
+    from openinference.instrumentation.agno import AgnoInstrumentor  # type: ignore
+    from opentelemetry import trace as trace_api  # type: ignore
+    from opentelemetry.sdk.trace import TracerProvider  # type: ignore
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanProcessor  # type: ignore
+
+    OPENTELEMETRY_AVAILABLE = True
+except ImportError:
+    OPENTELEMETRY_AVAILABLE = False
+
+
+def setup_tracing(
+    db: Union[BaseDb, AsyncBaseDb],
+    batch_processing: bool = False,
+    max_queue_size: int = 2048,
+    max_export_batch_size: int = 512,
+    schedule_delay_millis: int = 5000,
+) -> None:
+    """
+    Set up OpenTelemetry tracing with database export for Agno agents.
+
+    This function configures automatic tracing for all Agno agents, teams, and workflows.
+    Traces are automatically captured for:
+    - Agent runs (agent.run, agent.arun)
+    - Model calls (model.response)
+    - Tool executions
+    - Team coordination
+    - Workflow steps
+
+    Args:
+        db: Database instance to store traces (sync or async)
+        batch_processing: If True, use BatchSpanProcessor for better performance
+                            If False, use SimpleSpanProcessor (immediate export)
+        max_queue_size: Maximum queue size for batch processor
+        max_export_batch_size: Maximum batch size for export
+        schedule_delay_millis: Delay in milliseconds between batch exports
+
+    Raises:
+        ImportError: If OpenTelemetry packages are not installed
+
+    Example:
+        ```python
+        from agno.db.sqlite import SqliteDb
+        from agno.tracing import setup_tracing
+
+        db = SqliteDb(db_file="tmp/traces.db")
+        setup_tracing(db=db)
+
+        # Now all agents will be automatically traced
+        agent = Agent(...)
+        agent.run("Hello")  # This will be traced automatically
+        ```
+    """
+    if not OPENTELEMETRY_AVAILABLE:
+        raise ImportError(
+            "OpenTelemetry packages are required for tracing. "
+            "Install with: pip install opentelemetry-api opentelemetry-sdk openinference-instrumentation-agno"
+        )
+
+    # Check if tracing is already set up (handles reload scenarios)
+    current_provider = trace_api.get_tracer_provider()
+    if isinstance(current_provider, TracerProvider):
+        # Already configured with a real TracerProvider, skip
+        return
+
+    try:
+        # Create tracer provider
+        tracer_provider = TracerProvider()
+
+        # Create database exporter
+        exporter = DatabaseSpanExporter(db=db)
+
+        # Configure span processor
+        processor: SpanProcessor
+        if batch_processing:
+            processor = BatchSpanProcessor(
+                exporter,
+                max_queue_size=max_queue_size,
+                max_export_batch_size=max_export_batch_size,
+                schedule_delay_millis=schedule_delay_millis,
+            )
+            logger.debug(
+                f"Tracing configured with BatchSpanProcessor "
+                f"(queue_size={max_queue_size}, batch_size={max_export_batch_size})"
+            )
+        else:
+            processor = SimpleSpanProcessor(exporter)
+            logger.debug("Tracing configured with SimpleSpanProcessor")
+
+        tracer_provider.add_span_processor(processor)
+
+        # Set the global tracer provider
+        trace_api.set_tracer_provider(tracer_provider)
+
+        # Instrument Agno with OpenInference
+        AgnoInstrumentor().instrument(tracer_provider=tracer_provider)
+
+        logger.info("Agno tracing successfully set up with database storage")
+    except Exception as e:
+        logger.error(f"Failed to set up tracing: {e}", exc_info=True)
+        raise

agno/utils/agent.py

@@ -24,7 +24,7 @@ if TYPE_CHECKING:
     from agno.team.team import Team
 
 
-async def await_for_background_tasks(
+async def await_for_open_threads(
     memory_task: Optional[Task] = None,
     cultural_knowledge_task: Optional[Task] = None,
 ) -> None:
@@ -41,7 +41,7 @@ async def await_for_background_tasks(
             log_warning(f"Error in cultural knowledge creation: {str(e)}")
 
 
-def wait_for_background_tasks(
+def wait_for_open_threads(
     memory_future: Optional[Future] = None, cultural_knowledge_future: Optional[Future] = None
 ) -> None:
     if memory_future is not None:
@@ -58,7 +58,7 @@ def wait_for_background_tasks(
             log_warning(f"Error in cultural knowledge creation: {str(e)}")
 
 
-async def await_for_background_tasks_stream(
+async def await_for_thread_tasks_stream(
     run_response: Union[RunOutput, TeamRunOutput],
     memory_task: Optional[Task] = None,
     cultural_knowledge_task: Optional[Task] = None,
@@ -109,7 +109,7 @@ async def await_for_background_tasks_stream(
             log_warning(f"Error in cultural knowledge creation: {str(e)}")
 
 
-def wait_for_background_tasks_stream(
+def wait_for_thread_tasks_stream(
     run_response: Union[TeamRunOutput, RunOutput],
     memory_future: Optional[Future] = None,
     cultural_knowledge_future: Optional[Future] = None,
@@ -843,7 +843,7 @@ def execute_instructions(
 
     # Check for session_state parameter
     if "session_state" in signature.parameters:
-        instruction_args["session_state"] = session_state or {}
+        instruction_args["session_state"] = session_state if session_state is not None else {}
 
     # Check for run_context parameter
     if "run_context" in signature.parameters:
@@ -902,7 +902,7 @@ async def aexecute_instructions(
 
     # Check for session_state parameter
     if "session_state" in signature.parameters:
-        instruction_args["session_state"] = session_state or {}
+        instruction_args["session_state"] = session_state if session_state is not None else {}
 
     # Check for run_context parameter
     if "run_context" in signature.parameters:

agno/utils/hooks.py

@@ -1,14 +1,69 @@
+from copy import deepcopy
 from typing import Any, Callable, Dict, List, Optional, Union
 
 from agno.guardrails.base import BaseGuardrail
+from agno.hooks.decorator import HOOK_RUN_IN_BACKGROUND_ATTR
 from agno.utils.log import log_warning
 
+# Keys that should be deep copied for background hooks to prevent race conditions
+BACKGROUND_HOOK_COPY_KEYS = frozenset(
+    {"run_input", "run_context", "run_output", "session_state", "dependencies", "metadata"}
+)
+
+
+def copy_args_for_background(args: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Create a copy of hook arguments for background execution.
+
+    This deep copies run_input, run_context, run_output, session_state, dependencies,
+    and metadata to prevent race conditions when hooks run in the background.
+
+    Args:
+        args: The original arguments dictionary
+
+    Returns:
+        A new dictionary with copied values for sensitive keys
+    """
+    copied_args = {}
+    for key, value in args.items():
+        if key in BACKGROUND_HOOK_COPY_KEYS and value is not None:
+            try:
+                copied_args[key] = deepcopy(value)
+            except Exception:
+                # If deepcopy fails (e.g., for non-copyable objects), use the original
+                log_warning(f"Could not deepcopy {key} for background hook, using original reference")
+                copied_args[key] = value
+        else:
+            copied_args[key] = value
+    return copied_args
+
+
+def should_run_hook_in_background(hook: Callable[..., Any]) -> bool:
+    """
+    Check if a hook function should run in background.
+
+    This checks for the _agno_run_in_background attribute set by the @hook decorator.
+
+    Args:
+        hook: The hook function to check
+
+    Returns:
+        True if the hook is decorated with @hook(run_in_background=True)
+    """
+    return getattr(hook, HOOK_RUN_IN_BACKGROUND_ATTR, False)
+
 
 def normalize_hooks(
     hooks: Optional[List[Union[Callable[..., Any], BaseGuardrail]]],
     async_mode: bool = False,
 ) -> Optional[List[Callable[..., Any]]]:
-    """Normalize hooks to a list format"""
+    """Normalize hooks to a list format
+
+    Args:
+        hooks: List of hook functions or hook instances
+        async_mode: Whether to use async versions of methods
+    """
+
     result_hooks: List[Callable[..., Any]] = []
 
     if hooks is not None:

agno/utils/mcp.py

@@ -32,7 +32,7 @@ def get_entrypoint_for_tool(tool: MCPTool, session: ClientSession):
         try:
             await session.send_ping()
         except Exception as e:
-            print(e)
+            log_exception(e)
 
         try:
             log_debug(f"Calling MCP Tool '{tool_name}' with args: {kwargs}")