agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/models/openai/responses.py

@@ -8,16 +8,20 @@ from typing_extensions import Literal
 
 from agno.exceptions import ModelProviderError
 from agno.media import File
-from agno.models.base import MessageData, Model, _add_usage_metrics_to_assistant_message
+from agno.models.base import MessageData, Model
 from agno.models.message import Citations, Message, UrlCitation
+from agno.models.metrics import Metrics
 from agno.models.response import ModelResponse
+from agno.run.agent import RunOutput
 from agno.utils.log import log_debug, log_error, log_warning
 from agno.utils.models.openai_responses import images_to_message
 from agno.utils.models.schema_utils import get_response_schema_for_provider
 
 try:
     from openai import APIConnectionError, APIStatusError, AsyncOpenAI, OpenAI, RateLimitError
-    from openai.resources.responses.responses import Response, ResponseStreamEvent
+    from openai.types.responses.response import Response
+    from openai.types.responses.response_stream_event import ResponseStreamEvent
+    from openai.types.responses.response_usage import ResponseUsage
 except (ImportError, ModuleNotFoundError) as e:
     raise ImportError("`openai` not installed. Please install using `pip install openai -U`") from e
 
@@ -476,10 +480,12 @@ class OpenAIResponses(Model):
     def invoke(
         self,
         messages: List[Message],
+        assistant_message: Message,
         response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Response:
+        run_response: Optional[RunOutput] = None,
+    ) -> ModelResponse:
         """
         Send a request to the OpenAI Responses API.
         """
@@ -487,11 +493,24 @@ class OpenAIResponses(Model):
             request_params = self.get_request_params(
                 messages=messages, response_format=response_format, tools=tools, tool_choice=tool_choice
             )
-            return self.get_client().responses.create(
+
+            if run_response and run_response.metrics:
+                run_response.metrics.set_time_to_first_token()
+
+            assistant_message.metrics.start_timer()
+
+            provider_response = self.get_client().responses.create(
                 model=self.id,
                 input=self._format_messages(messages),  # type: ignore
                 **request_params,
             )
+
+            assistant_message.metrics.stop_timer()
+
+            model_response = self._parse_provider_response(provider_response, response_format=response_format)
+
+            return model_response
+
         except RateLimitError as exc:
             log_error(f"Rate limit error from OpenAI API: {exc}")
             error_message = exc.response.json().get("error", {})
@@ -530,10 +549,12 @@ class OpenAIResponses(Model):
     async def ainvoke(
         self,
         messages: List[Message],
+        assistant_message: Message,
         response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Response:
+        run_response: Optional[RunOutput] = None,
+    ) -> ModelResponse:
         """
         Sends an asynchronous request to the OpenAI Responses API.
         """
@@ -542,11 +563,23 @@ class OpenAIResponses(Model):
                 messages=messages, response_format=response_format, tools=tools, tool_choice=tool_choice
             )
 
-            return await self.get_async_client().responses.create(
+            if run_response and run_response.metrics:
+                run_response.metrics.set_time_to_first_token()
+
+            assistant_message.metrics.start_timer()
+
+            provider_response = await self.get_async_client().responses.create(
                 model=self.id,
                 input=self._format_messages(messages),  # type: ignore
                 **request_params,
             )
+
+            assistant_message.metrics.stop_timer()
+
+            model_response = self._parse_provider_response(provider_response, response_format=response_format)
+
+            return model_response
+
         except RateLimitError as exc:
             log_error(f"Rate limit error from OpenAI API: {exc}")
             error_message = exc.response.json().get("error", {})
@@ -585,10 +618,12 @@ class OpenAIResponses(Model):
     def invoke_stream(
         self,
         messages: List[Message],
+        assistant_message: Message,
         response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Iterator[ResponseStreamEvent]:
+        run_response: Optional[RunOutput] = None,
+    ) -> Iterator[ModelResponse]:
         """
         Send a streaming request to the OpenAI Responses API.
         """
@@ -596,13 +631,28 @@ class OpenAIResponses(Model):
             request_params = self.get_request_params(
                 messages=messages, response_format=response_format, tools=tools, tool_choice=tool_choice
             )
+            tool_use: Dict[str, Any] = {}
+
+            if run_response and run_response.metrics:
+                run_response.metrics.set_time_to_first_token()
+
+            assistant_message.metrics.start_timer()
 
-            yield from self.get_client().responses.create(
+            for chunk in self.get_client().responses.create(
                 model=self.id,
                 input=self._format_messages(messages),  # type: ignore
                 stream=True,
                 **request_params,
-            )  # type: ignore
+            ):
+                model_response, tool_use = self._parse_provider_response_delta(
+                    stream_event=chunk,  # type: ignore
+                    assistant_message=assistant_message,
+                    tool_use=tool_use,  # type: ignore
+                )
+                yield model_response
+
+            assistant_message.metrics.stop_timer()
+
         except RateLimitError as exc:
             log_error(f"Rate limit error from OpenAI API: {exc}")
             error_message = exc.response.json().get("error", {})
@@ -641,10 +691,12 @@ class OpenAIResponses(Model):
     async def ainvoke_stream(
         self,
         messages: List[Message],
+        assistant_message: Message,
         response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
         tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> AsyncIterator[ResponseStreamEvent]:
+        run_response: Optional[RunOutput] = None,
+    ) -> AsyncIterator[ModelResponse]:
         """
         Sends an asynchronous streaming request to the OpenAI Responses API.
         """
@@ -652,6 +704,13 @@ class OpenAIResponses(Model):
             request_params = self.get_request_params(
                 messages=messages, response_format=response_format, tools=tools, tool_choice=tool_choice
             )
+            tool_use: Dict[str, Any] = {}
+
+            if run_response and run_response.metrics:
+                run_response.metrics.set_time_to_first_token()
+
+            assistant_message.metrics.start_timer()
+
             async_stream = await self.get_async_client().responses.create(
                 model=self.id,
                 input=self._format_messages(messages),  # type: ignore
@@ -659,7 +718,11 @@ class OpenAIResponses(Model):
                 **request_params,
             )
             async for chunk in async_stream:  # type: ignore
-                yield chunk
+                model_response, tool_use = self._parse_provider_response_delta(chunk, assistant_message, tool_use)  # type: ignore
+                yield model_response
+
+            assistant_message.metrics.stop_timer()
+
         except RateLimitError as exc:
             log_error(f"Rate limit error from OpenAI API: {exc}")
             error_message = exc.response.json().get("error", {})
@@ -711,7 +774,64 @@ class OpenAIResponses(Model):
                 _fc_message.tool_call_id = tool_call_ids[_fc_message_index]
                 messages.append(_fc_message)
 
-    def parse_provider_response(self, response: Response, **kwargs) -> ModelResponse:
+    def process_response_stream(
+        self,
+        messages: List[Message],
+        assistant_message: Message,
+        stream_data: MessageData,
+        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
+        run_response: Optional[RunOutput] = None,
+    ) -> Iterator[ModelResponse]:
+        """Process the synchronous response stream."""
+        for model_response_delta in self.invoke_stream(
+            messages=messages,
+            assistant_message=assistant_message,
+            tools=tools,
+            response_format=response_format,
+            tool_choice=tool_choice,
+            run_response=run_response,
+        ):
+            yield from self._populate_stream_data_and_assistant_message(
+                stream_data=stream_data,
+                assistant_message=assistant_message,
+                model_response_delta=model_response_delta,
+            )
+
+        # Add final metrics to assistant message
+        self._populate_assistant_message(assistant_message=assistant_message, provider_response=model_response_delta)
+
+    async def aprocess_response_stream(
+        self,
+        messages: List[Message],
+        assistant_message: Message,
+        stream_data: MessageData,
+        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
+        run_response: Optional[RunOutput] = None,
+    ) -> AsyncIterator[ModelResponse]:
+        """Process the asynchronous response stream."""
+        async for model_response_delta in self.ainvoke_stream(
+            messages=messages,
+            assistant_message=assistant_message,
+            tools=tools,
+            response_format=response_format,
+            tool_choice=tool_choice,
+            run_response=run_response,
+        ):
+            for model_response in self._populate_stream_data_and_assistant_message(
+                stream_data=stream_data,
+                assistant_message=assistant_message,
+                model_response_delta=model_response_delta,
+            ):
+                yield model_response
+
+        # Add final metrics to assistant message
+        self._populate_assistant_message(assistant_message=assistant_message, provider_response=model_response_delta)
+
+    def _parse_provider_response(self, response: Response, **kwargs) -> ModelResponse:
         """
         Parse the OpenAI response into a ModelResponse.
 
@@ -797,76 +917,66 @@ class OpenAIResponses(Model):
 
         # Add metrics
         if response.usage is not None:
-            model_response.response_usage = response.usage
+            model_response.response_usage = self._get_metrics(response.usage)
 
         return model_response
 
-    def _process_stream_response(
-        self,
-        stream_event: ResponseStreamEvent,
-        assistant_message: Message,
-        stream_data: MessageData,
-        tool_use: Dict[str, Any],
-    ) -> Tuple[Optional[ModelResponse], Dict[str, Any]]:
+    def _parse_provider_response_delta(
+        self, stream_event: ResponseStreamEvent, assistant_message: Message, tool_use: Dict[str, Any]
+    ) -> Tuple[ModelResponse, Dict[str, Any]]:
         """
-        Common handler for processing stream responses from Cohere.
+        Parse the streaming response from the model provider into a ModelResponse object.
 
         Args:
-            stream_event: The streamed response from Cohere
-            assistant_message: The assistant message being built
-            stream_data: Data accumulated during streaming
-            tool_use: Current tool use data being built
+            response: Raw response chunk from the model provider
 
         Returns:
-            Tuple containing the ModelResponse to yield and updated tool_use dict
+            ModelResponse: Parsed response delta
         """
-        model_response = None
+        model_response = ModelResponse()
 
+        # 1. Add response ID
         if stream_event.type == "response.created":
-            model_response = ModelResponse()
-            # Store the response ID for continuity
             if stream_event.response.id:
-                if stream_data.response_provider_data is None:
-                    stream_data.response_provider_data = {}
-                stream_data.response_provider_data["response_id"] = stream_event.response.id
-            # Update metrics
+                if model_response.provider_data is None:
+                    model_response.provider_data = {}
+                model_response.provider_data["response_id"] = stream_event.response.id
             if not assistant_message.metrics.time_to_first_token:
                 assistant_message.metrics.set_time_to_first_token()
+
+        # 2. Add citations
         elif stream_event.type == "response.output_text.annotation.added":
-            model_response = ModelResponse()
-            if stream_data.response_citations is None:
-                stream_data.response_citations = Citations(raw=[stream_event.annotation])
+            if model_response.citations is None:
+                model_response.citations = Citations(raw=[stream_event.annotation])
             else:
-                stream_data.response_citations.raw.append(stream_event.annotation)  # type: ignore
+                model_response.citations.raw.append(stream_event.annotation)  # type: ignore
 
             if isinstance(stream_event.annotation, dict):
                 if stream_event.annotation.get("type") == "url_citation":
-                    if stream_data.response_citations.urls is None:
-                        stream_data.response_citations.urls = []
-                    stream_data.response_citations.urls.append(
+                    if model_response.citations.urls is None:
+                        model_response.citations.urls = []
+                    model_response.citations.urls.append(
                         UrlCitation(url=stream_event.annotation.get("url"), title=stream_event.annotation.get("title"))
                     )
             else:
                 if stream_event.annotation.type == "url_citation":  # type: ignore
-                    if stream_data.response_citations.urls is None:
-                        stream_data.response_citations.urls = []
-                    stream_data.response_citations.urls.append(
+                    if model_response.citations.urls is None:
+                        model_response.citations.urls = []
+                    model_response.citations.urls.append(
                         UrlCitation(url=stream_event.annotation.url, title=stream_event.annotation.title)  # type: ignore
                     )
 
-            model_response.citations = stream_data.response_citations
-
+        # 3. Add content
         elif stream_event.type == "response.output_text.delta":
-            model_response = ModelResponse()
-            # Add content
             model_response.content = stream_event.delta
-            stream_data.response_content += stream_event.delta
 
             # Treat the output_text deltas as reasoning content if the reasoning summary is not requested.
             if self.reasoning is not None and self.reasoning_summary is None:
                 model_response.reasoning_content = stream_event.delta
-                stream_data.response_thinking += stream_event.delta
 
+        # 4. Add tool calls information
+
+        # 4.1 Add starting tool call
         elif stream_event.type == "response.output_item.added":
             item = stream_event.item
             if item.type == "function_call":
@@ -880,20 +990,22 @@ class OpenAIResponses(Model):
                     },
                 }
 
+        # 4.2 Add tool call arguments
         elif stream_event.type == "response.function_call_arguments.delta":
             tool_use["function"]["arguments"] += stream_event.delta
 
+        # 4.3 Add tool call completion data
         elif stream_event.type == "response.output_item.done" and tool_use:
-            model_response = ModelResponse()
             model_response.tool_calls = [tool_use]
             if assistant_message.tool_calls is None:
                 assistant_message.tool_calls = []
             assistant_message.tool_calls.append(tool_use)
 
-            stream_data.extra = stream_data.extra or {}
-            stream_data.extra.setdefault("tool_call_ids", []).append(tool_use["call_id"])
+            model_response.extra = model_response.extra or {}
+            model_response.extra.setdefault("tool_call_ids", []).append(tool_use["call_id"])
             tool_use = {}
 
+        # 5. Add metrics
         elif stream_event.type == "response.completed":
             model_response = ModelResponse()
 
@@ -915,63 +1027,24 @@ class OpenAIResponses(Model):
 
             # Add metrics
             if stream_event.response.usage is not None:
-                model_response.response_usage = stream_event.response.usage
-
-            _add_usage_metrics_to_assistant_message(
-                assistant_message=assistant_message,
-                response_usage=model_response.response_usage,
-            )
+                model_response.response_usage = self._get_metrics(stream_event.response.usage)
 
         return model_response, tool_use
 
-    def process_response_stream(
-        self,
-        messages: List[Message],
-        assistant_message: Message,
-        stream_data: MessageData,
-        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> Iterator[ModelResponse]:
-        """Process the synchronous response stream."""
-        tool_use: Dict[str, Any] = {}
-
-        for stream_event in self.invoke_stream(
-            messages=messages, tools=tools, response_format=response_format, tool_choice=tool_choice
-        ):
-            model_response, tool_use = self._process_stream_response(
-                stream_event=stream_event,
-                assistant_message=assistant_message,
-                stream_data=stream_data,
-                tool_use=tool_use,
-            )
+    def _get_metrics(self, response_usage: ResponseUsage) -> Metrics:
+        """
+        Parse the given OpenAI-specific usage into an Agno Metrics object.
 
-            if model_response is not None:
-                yield model_response
+        Args:
+            response: The response from the provider.
 
-    async def aprocess_response_stream(
-        self,
-        messages: List[Message],
-        assistant_message: Message,
-        stream_data: MessageData,
-        response_format: Optional[Union[Dict, Type[BaseModel]]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
-    ) -> AsyncIterator[ModelResponse]:
-        """Process the asynchronous response stream."""
-        tool_use: Dict[str, Any] = {}
+        Returns:
+            Metrics: Parsed metrics data
+        """
+        metrics = Metrics()
 
-        async for stream_event in self.ainvoke_stream(
-            messages=messages, tools=tools, response_format=response_format, tool_choice=tool_choice
-        ):
-            model_response, tool_use = self._process_stream_response(
-                stream_event=stream_event,
-                assistant_message=assistant_message,
-                stream_data=stream_data,
-                tool_use=tool_use,
-            )
-            if model_response is not None:
-                yield model_response
+        metrics.input_tokens = response_usage.input_tokens or 0
+        metrics.output_tokens = response_usage.output_tokens or 0
+        metrics.total_tokens = response_usage.total_tokens or 0
 
-    def parse_provider_response_delta(self, response: Any) -> ModelResponse:  # type: ignore
-        pass
+        return metrics

agno/models/perplexity/perplexity.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel
 
 from agno.exceptions import ModelProviderError
 from agno.models.message import Citations, UrlCitation
+from agno.models.metrics import Metrics
 from agno.models.response import ModelResponse
 from agno.utils.log import log_debug, log_warning
 
@@ -16,6 +17,7 @@ try:
         ChoiceDelta,
     )
     from openai.types.chat.parsed_chat_completion import ParsedChatCompletion
+    from openai.types.completion_usage import CompletionUsage
 except ModuleNotFoundError:
     raise ImportError("`openai` not installed. Please install using `pip install openai`")
 
@@ -124,7 +126,7 @@ class Perplexity(OpenAILike):
             )
 
         if response.usage is not None:
-            model_response.response_usage = response.usage
+            model_response.response_usage = self._get_metrics(response.usage)
 
         return model_response
 
@@ -159,6 +161,28 @@ class Perplexity(OpenAILike):
 
         # Add usage metrics if present
         if response_delta.usage is not None:
-            model_response.response_usage = response_delta.usage
+            model_response.response_usage = self._get_metrics(response_delta.usage)
 
         return model_response
+
+    def _get_metrics(self, response_usage: CompletionUsage) -> Metrics:
+        """
+        Parse the given Perplexity usage into an Agno Metrics object.
+        """
+        metrics = Metrics()
+
+        metrics.input_tokens = response_usage.prompt_tokens or 0
+        metrics.output_tokens = response_usage.completion_tokens or 0
+        metrics.total_tokens = response_usage.total_tokens or 0
+
+        # Add the prompt_tokens_details field
+        if prompt_token_details := response_usage.prompt_tokens_details:
+            metrics.audio_input_tokens = prompt_token_details.audio_tokens or 0
+            metrics.cache_read_tokens = prompt_token_details.cached_tokens or 0
+
+        # Add the completion_tokens_details field
+        if completion_tokens_details := response_usage.completion_tokens_details:
+            metrics.audio_output_tokens = completion_tokens_details.audio_tokens or 0
+            metrics.reasoning_tokens = completion_tokens_details.reasoning_tokens or 0
+
+        return metrics

agno/models/portkey/portkey.py

@@ -44,13 +44,6 @@ class Portkey(OpenAILike):
                 model_id=self.id,
             )
 
-        if not self.virtual_key:
-            raise ModelProviderError(
-                message="PORTKEY_VIRTUAL_KEY not set. Please set the PORTKEY_VIRTUAL_KEY environment variable.",
-                model_name=self.name,
-                model_id=self.id,
-            )
-
         # Create headers using Portkey's createHeaders function
         header_params: Dict[str, Any] = {
             "api_key": self.portkey_api_key,

agno/models/response.py

@@ -3,8 +3,9 @@ from enum import Enum
 from time import time
 from typing import Any, Dict, List, Optional
 
-from agno.media import AudioResponse, ImageArtifact
-from agno.models.message import Citations, MessageMetrics
+from agno.media import AudioArtifact, AudioResponse, ImageArtifact, VideoArtifact
+from agno.models.message import Citations
+from agno.models.metrics import Metrics
 from agno.tools.function import UserInputField
 
 
@@ -26,7 +27,7 @@ class ToolExecution:
     tool_args: Optional[Dict[str, Any]] = None
     tool_call_error: Optional[bool] = None
     result: Optional[str] = None
-    metrics: Optional[MessageMetrics] = None
+    metrics: Optional[Metrics] = None
 
     # If True, the agent will stop executing after this tool call.
     stop_after_tool_call: bool = False
@@ -74,7 +75,7 @@ class ToolExecution:
             if "user_input_schema" in data
             else None,
             external_execution_required=data.get("external_execution_required"),
-            metrics=MessageMetrics(**(data.get("metrics", {}) or {})),
+            metrics=Metrics(**(data.get("metrics", {}) or {})),
         )
 
 
@@ -87,7 +88,11 @@ class ModelResponse:
     content: Optional[Any] = None
     parsed: Optional[Any] = None
     audio: Optional[AudioResponse] = None
-    image: Optional[ImageArtifact] = None
+
+    # Unified media fields for LLM-generated and tool-generated media artifacts
+    images: Optional[List[ImageArtifact]] = None
+    videos: Optional[List[VideoArtifact]] = None
+    audios: Optional[List[AudioArtifact]] = None
 
     # Model tool calls
     tool_calls: List[Dict[str, Any]] = field(default_factory=list)
@@ -99,18 +104,19 @@ class ModelResponse:
 
     provider_data: Optional[Dict[str, Any]] = None
 
-    thinking: Optional[str] = None
-    redacted_thinking: Optional[str] = None
+    redacted_reasoning_content: Optional[str] = None
     reasoning_content: Optional[str] = None
 
     citations: Optional[Citations] = None
 
-    response_usage: Optional[Any] = None
+    response_usage: Optional[Metrics] = None
 
     created_at: int = int(time())
 
     extra: Optional[Dict[str, Any]] = None
 
+    updated_session_state: Optional[Dict[str, Any]] = None
+
 
 class FileType(str, Enum):
     MP4 = "mp4"

agno/models/utils.py

@@ -0,0 +1,20 @@
+from agno.models.base import Model
+
+
+# TODO: add all supported models
+def get_model(model_id: str, model_provider: str) -> Model:
+    """Return the right Agno model instance given a pair of model provider and id"""
+    if model_provider == "openai":
+        from agno.models.openai import OpenAIChat
+
+        return OpenAIChat(id=model_id)
+    elif model_provider == "anthropic":
+        from agno.models.anthropic import Claude
+
+        return Claude(id=model_id)
+    elif model_provider == "gemini":
+        from agno.models.google import Gemini
+
+        return Gemini(id=model_id)
+    else:
+        raise ValueError(f"Model provider {model_provider} not supported")

agno/models/vercel/__init__.py

@@ -1,3 +1,3 @@
-from agno.models.vercel.v0 import v0
+from agno.models.vercel.v0 import V0
 
-__all__ = ["v0"]
+__all__ = ["V0"]

agno/models/vercel/v0.py

@@ -6,7 +6,7 @@ from agno.models.openai.like import OpenAILike
 
 
 @dataclass
-class v0(OpenAILike):
+class V0(OpenAILike):
     """
     Class for interacting with the v0 API.
 

agno/models/vllm/__init__.py

@@ -1,3 +1,3 @@
-from agno.models.vllm.vllm import vLLM
+from agno.models.vllm.vllm import VLLM
 
-__all__ = ["vLLM"]
+__all__ = ["VLLM"]

agno/models/vllm/vllm.py

@@ -9,7 +9,7 @@ from agno.utils.log import log_debug
 
 
 @dataclass
-class vLLM(OpenAILike):
+class VLLM(OpenAILike):
     """
     Class for interacting with vLLM models via OpenAI-compatible API.
 
@@ -26,8 +26,8 @@ class vLLM(OpenAILike):
     """
 
     id: str = "not-set"
-    name: str = "vLLM"
-    provider: str = "vLLM"
+    name: str = "VLLM"
+    provider: str = "VLLM"
 
     api_key: Optional[str] = getenv("VLLM_API_KEY") or "EMPTY"
     base_url: Optional[str] = getenv("VLLM_BASE_URL", "http://localhost:8000/v1/")