ag2 0.10.2__py3-none-any.whl

autogen/agentchat/realtime/experimental/clients/realtime_client.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+from collections.abc import AsyncGenerator, Callable
+from contextlib import AbstractAsyncContextManager
+from logging import Logger
+from typing import Any, Literal, Protocol, TypeVar, runtime_checkable
+
+from anyio import create_task_group
+
+from .....doc_utils import export_module
+from .....llm_config import LLMConfig
+from ..realtime_events import InputAudioBufferDelta, RealtimeEvent
+
+__all__ = ["RealtimeClientProtocol", "Role", "get_client", "register_realtime_client"]
+
+# define role literal type for typing
+Role = Literal["user", "assistant", "system"]
+
+
+@runtime_checkable
+@export_module("autogen.agentchat.realtime.experimental.clients")
+class RealtimeClientProtocol(Protocol):
+    async def send_function_result(self, call_id: str, result: str) -> None:
+        """Send the result of a function call to a Realtime API.
+
+        Args:
+            call_id (str): The ID of the function call.
+            result (str): The result of the function call.
+        """
+        ...
+
+    async def send_text(self, *, role: Role, text: str) -> None:
+        """Send a text message to a Realtime API.
+
+        Args:
+            role (str): The role of the message.
+            text (str): The text of the message.
+        """
+        ...
+
+    async def send_audio(self, audio: str) -> None:
+        """Send audio to a Realtime API.
+
+        Args:
+            audio (str): The audio to send.
+        """
+        ...
+
+    async def truncate_audio(self, audio_end_ms: int, content_index: int, item_id: str) -> None:
+        """Truncate audio in a Realtime API.
+
+        Args:
+            audio_end_ms (int): The end of the audio to truncate.
+            content_index (int): The index of the content to truncate.
+            item_id (str): The ID of the item to truncate.
+        """
+        ...
+
+    async def session_update(self, session_options: dict[str, Any]) -> None:
+        """Send a session update to a Realtime API.
+
+        Args:
+            session_options (dict[str, Any]): The session options to update.
+        """
+        ...
+
+    def connect(self) -> AbstractAsyncContextManager[None]: ...
+
+    def read_events(self) -> AsyncGenerator[RealtimeEvent, None]:
+        """Read events from a Realtime Client."""
+        ...
+
+    async def _read_from_connection(self) -> AsyncGenerator[RealtimeEvent, None]:
+        """Read events from a Realtime connection."""
+        ...
+
+    def _parse_message(self, message: dict[str, Any]) -> list[RealtimeEvent]:
+        """Parse a message from a Realtime API.
+
+        Args:
+            message (dict[str, Any]): The message to parse.
+
+        Returns:
+            list[RealtimeEvent]: The parsed events.
+        """
+        ...
+
+    @classmethod
+    def get_factory(
+        cls, llm_config: LLMConfig | dict[str, Any], logger: Logger, **kwargs: Any
+    ) -> Callable[[], "RealtimeClientProtocol"] | None:
+        """Create a Realtime API client.
+
+        Args:
+            llm_config: The config for the client.
+            logger: The logger to use for logging events.
+            **kwargs: Additional arguments.
+
+        Returns:
+            RealtimeClientProtocol: The Realtime API client is returned if the model matches the pattern
+        """
+        ...
+
+
+class RealtimeClientBase:
+    def __init__(self):
+        self._eventQueue = asyncio.Queue()
+
+    async def add_event(self, event: RealtimeEvent | None):
+        await self._eventQueue.put(event)
+
+    async def get_event(self) -> RealtimeEvent | None:
+        return await self._eventQueue.get()
+
+    async def _read_from_connection_task(self):
+        async for event in self._read_from_connection():
+            await self.add_event(event)
+        await self.add_event(None)
+
+    async def _read_events(self) -> AsyncGenerator[RealtimeEvent, None]:
+        """Read events from a Realtime Client."""
+        async with create_task_group() as tg:
+            tg.start_soon(self._read_from_connection_task)
+            while True:
+                try:
+                    event = await self._eventQueue.get()
+                    if event is not None:
+                        yield event
+                    else:
+                        break
+                except Exception:
+                    break
+
+    async def queue_input_audio_buffer_delta(self, audio: str) -> None:
+        """Queue InputAudioBufferDelta.
+
+        Args:
+            audio (str): The audio.
+        """
+        await self.add_event(InputAudioBufferDelta(delta=audio, item_id=None, raw_message={}))
+
+
+_realtime_client_classes: dict[str, type[RealtimeClientProtocol]] = {}
+
+T = TypeVar("T", bound=RealtimeClientProtocol)
+
+
+def register_realtime_client() -> Callable[[type[T]], type[T]]:
+    """Register a Realtime API client.
+
+    Returns:
+        Callable[[type[T]], type[T]]: The decorator to register the Realtime API client
+    """
+
+    def decorator(client_cls: type[T]) -> type[T]:
+        """Register a Realtime API client.
+
+        Args:
+            client_cls: The client to register.
+        """
+        global _realtime_client_classes
+        fqn = f"{client_cls.__module__}.{client_cls.__name__}"
+        _realtime_client_classes[fqn] = client_cls
+
+        return client_cls
+
+    return decorator
+
+
+@export_module("autogen.agentchat.realtime.experimental.clients")
+def get_client(llm_config: LLMConfig | dict[str, Any], logger: Logger, **kwargs: Any) -> "RealtimeClientProtocol":
+    """Get a registered Realtime API client.
+
+    Args:
+        llm_config: The config for the client.
+        logger: The logger to use for logging events.
+        **kwargs: Additional arguments.
+
+    Returns:
+        RealtimeClientProtocol: The Realtime API client.
+    """
+    global _realtime_client_classes
+    for _, client_cls in _realtime_client_classes.items():
+        factory = client_cls.get_factory(llm_config=llm_config, logger=logger, **kwargs)
+        if factory:
+            return factory()
+
+    raise ValueError("Realtime API client not found.")

autogen/agentchat/realtime/experimental/function_observer.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from typing import TYPE_CHECKING, Any, Optional
+
+from pydantic import BaseModel
+
+from ....doc_utils import export_module
+from ....fast_depends.utils import asyncify
+from .realtime_events import FunctionCall, RealtimeEvent
+from .realtime_observer import RealtimeObserver
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+
+@export_module("autogen.agentchat.realtime.experimental")
+class FunctionObserver(RealtimeObserver):
+    """Observer for handling function calls from the OpenAI Realtime API."""
+
+    def __init__(self, *, logger: Optional["Logger"] = None) -> None:
+        """Observer for handling function calls from the OpenAI Realtime API."""
+        super().__init__(logger=logger)
+
+    async def on_event(self, event: RealtimeEvent) -> None:
+        """Handle function call events from the OpenAI Realtime API.
+
+        Args:
+            event (dict[str, Any]): The event from the OpenAI Realtime API.
+        """
+        if isinstance(event, FunctionCall):
+            self.logger.info("Received function call event")
+            await self.call_function(
+                call_id=event.call_id,
+                name=event.name,
+                kwargs=event.arguments,
+            )
+
+    async def call_function(self, call_id: str, name: str, kwargs: dict[str, Any]) -> None:
+        """Call a function registered with the agent.
+
+        Args:
+            call_id (str): The ID of the function call.
+            name (str): The name of the function to call.
+            kwargs (Any[str, Any]): The arguments to pass to the function.
+        """
+        if name in self.agent.registered_realtime_tools:
+            func = self.agent.registered_realtime_tools[name].func
+            func = asyncify(func)
+            try:
+                result = await func(**kwargs)
+            except Exception:
+                result = "Function call failed"
+                self.logger.info(f"Function call failed: {name=}, {kwargs=}", stack_info=True)
+
+            if isinstance(result, BaseModel):
+                result = result.model_dump_json()
+            elif not isinstance(result, str):
+                try:
+                    result = json.dumps(result)
+                except Exception:
+                    result = str(result)
+
+            await self.realtime_client.send_function_result(call_id, result)
+        else:
+            self.logger.warning(f"Function {name} called, but is not registered with the realtime agent.")
+
+    async def initialize_session(self) -> None:
+        """Add registered tools to OpenAI with a session update."""
+        session_update = {
+            "tools": [tool.realtime_tool_schema for tool in self.agent.registered_realtime_tools.values()],
+            "tool_choice": "auto",
+        }
+        await self.realtime_client.session_update(session_update)
+
+    async def run_loop(self) -> None:
+        """Run the observer loop."""
+        pass
+
+
+if TYPE_CHECKING:
+    function_observer: RealtimeObserver = FunctionObserver()

autogen/agentchat/realtime/experimental/realtime_agent.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from logging import Logger, getLogger
+from typing import Any, TypeVar
+
+from anyio import create_task_group, lowlevel
+
+from ....doc_utils import export_module
+from ....llm_config import LLMConfig
+from ....tools import Tool
+from .clients.realtime_client import RealtimeClientProtocol, get_client
+from .function_observer import FunctionObserver
+from .realtime_observer import RealtimeObserver
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+global_logger = getLogger(__name__)
+
+
+@dataclass
+class RealtimeAgentCallbacks:
+    """Callbacks for the Realtime Agent."""
+
+    # async empty placeholder function
+    on_observers_ready: Callable[[], Any] = lambda: lowlevel.checkpoint()
+
+
+@export_module("autogen.agentchat.realtime.experimental")
+class RealtimeAgent:
+    def __init__(
+        self,
+        *,
+        name: str,
+        audio_adapter: RealtimeObserver | None = None,
+        system_message: str = "You are a helpful AI Assistant.",
+        llm_config: LLMConfig | dict[str, Any] | None = None,
+        logger: Logger | None = None,
+        observers: list[RealtimeObserver] | None = None,
+        **client_kwargs: Any,
+    ):
+        """(Experimental) Agent for interacting with the Realtime Clients.
+
+        Args:
+            name (str): The name of the agent.
+            audio_adapter (Optional[RealtimeObserver] = None): The audio adapter for the agent.
+            system_message (str): The system message for the agent.
+            llm_config (LLMConfig, dict[str, Any], bool): The config for the agent.
+            logger (Optional[Logger]): The logger for the agent.
+            observers (Optional[list[RealtimeObserver]]): The additional observers for the agent.
+            **client_kwargs (Any): The keyword arguments for the client.
+        """
+        self._logger = logger
+        self._name = name
+        self._system_message = system_message
+
+        llm_config = LLMConfig.get_current_llm_config(llm_config)
+
+        self._realtime_client: RealtimeClientProtocol = get_client(
+            llm_config=llm_config, logger=self.logger, **client_kwargs
+        )
+
+        self._registered_realtime_tools: dict[str, Tool] = {}
+        self._observers: list[RealtimeObserver] = observers if observers else []
+        self._observers.append(FunctionObserver(logger=logger))
+        if audio_adapter:
+            self._observers.append(audio_adapter)
+
+        self.callbacks = RealtimeAgentCallbacks()
+
+    @property
+    def system_message(self) -> str:
+        """Get the system message for the agent."""
+        return self._system_message
+
+    @property
+    def logger(self) -> Logger:
+        """Get the logger for the agent."""
+        return self._logger or global_logger
+
+    @property
+    def realtime_client(self) -> RealtimeClientProtocol:
+        """Get the OpenAI Realtime Client."""
+        return self._realtime_client
+
+    @property
+    def registered_realtime_tools(self) -> dict[str, Tool]:
+        """Get the registered realtime tools."""
+        return self._registered_realtime_tools
+
+    def register_observer(self, observer: RealtimeObserver) -> None:
+        """Register an observer with the Realtime Agent.
+
+        Args:
+            observer (RealtimeObserver): The observer to register.
+        """
+        self._observers.append(observer)
+
+    async def start_observers(self) -> None:
+        for observer in self._observers:
+            self._tg.start_soon(observer.run, self)
+
+        # wait for the observers to be ready
+        for observer in self._observers:
+            await observer.wait_for_ready()
+
+        await self.callbacks.on_observers_ready()
+
+    async def run(self) -> None:
+        """Run the agent."""
+        # everything is run in the same task group to enable easy cancellation using self._tg.cancel_scope.cancel()
+        async with create_task_group() as self._tg:  # noqa: SIM117
+            # connect with the client first (establishes a connection and initializes a session)
+            async with self._realtime_client.connect():
+                # start the observers and wait for them to be ready
+                await self.realtime_client.session_update(session_options={"instructions": self.system_message})
+                await self.start_observers()
+
+                # iterate over the events
+                async for event in self.realtime_client.read_events():
+                    for observer in self._observers:
+                        await observer.on_event(event)
+
+    def register_realtime_function(
+        self,
+        *,
+        name: str | None = None,
+        description: str | None = None,
+    ) -> Callable[[F | Tool], Tool]:
+        """Decorator for registering a function to be used by an agent.
+
+        Args:
+            name (str): The name of the function.
+            description (str): The description of the function.
+
+        Returns:
+            Callable[[Union[F, Tool]], Tool]: The decorator for registering a function.
+        """
+
+        def _decorator(func_or_tool: F | Tool) -> Tool:
+            """Decorator for registering a function to be used by an agent.
+
+            Args:
+                func_or_tool (Union[F, Tool]): The function or tool to register.
+
+            Returns:
+                Tool: The registered tool.
+            """
+            tool = Tool(func_or_tool=func_or_tool, name=name, description=description)
+
+            self._registered_realtime_tools[tool.name] = tool
+
+            return tool
+
+        return _decorator

autogen/agentchat/realtime/experimental/realtime_events.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+
+class RealtimeEvent(BaseModel):
+    raw_message: dict[str, Any]
+
+
+class SessionCreated(RealtimeEvent):
+    type: Literal["session.created"] = "session.created"
+
+
+class SessionUpdated(RealtimeEvent):
+    type: Literal["session.updated"] = "session.updated"
+
+
+class AudioDelta(RealtimeEvent):
+    type: Literal["response.audio.delta"] = "response.audio.delta"
+    delta: str
+    item_id: Any
+
+
+class InputAudioBufferDelta(RealtimeEvent):
+    type: Literal["input_audio_buffer.delta"] = "input_audio_buffer.delta"
+    delta: str
+    item_id: Any
+
+
+class SpeechStarted(RealtimeEvent):
+    type: Literal["input_audio_buffer.speech_started"] = "input_audio_buffer.speech_started"
+
+
+class FunctionCall(RealtimeEvent):
+    type: Literal["response.function_call_arguments.done"] = "response.function_call_arguments.done"
+    name: str
+    arguments: dict[str, Any]
+    call_id: str

autogen/agentchat/realtime/experimental/realtime_observer.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from abc import ABC, abstractmethod
+from logging import Logger, getLogger
+from typing import TYPE_CHECKING
+
+from anyio import Event
+
+from ....doc_utils import export_module
+from .clients.realtime_client import RealtimeClientProtocol
+from .realtime_events import RealtimeEvent
+
+if TYPE_CHECKING:
+    from .realtime_agent import RealtimeAgent
+
+__all__ = ["RealtimeObserver"]
+
+global_logger = getLogger(__name__)
+
+
+@export_module("autogen.agentchat.realtime.experimental")
+class RealtimeObserver(ABC):
+    """Observer for the OpenAI Realtime API."""
+
+    def __init__(self, *, logger: Logger | None = None) -> None:
+        """Observer for the OpenAI Realtime API.
+
+        Args:
+            logger (Logger): The logger for the observer.
+        """
+        self._ready_event = Event()
+        self._agent: RealtimeAgent | None = None
+        self._logger = logger
+
+    @property
+    def logger(self) -> Logger:
+        return self._logger or global_logger
+
+    @property
+    def agent(self) -> "RealtimeAgent":
+        if self._agent is None:
+            raise RuntimeError("Agent has not been set.")
+        return self._agent
+
+    @property
+    def realtime_client(self) -> RealtimeClientProtocol:
+        if self._agent is None:
+            raise RuntimeError("Agent has not been set.")
+        if self._agent.realtime_client is None:
+            raise RuntimeError("Realtime client has not been set.")
+
+        return self._agent.realtime_client
+
+    async def run(self, agent: "RealtimeAgent") -> None:
+        """Run the observer with the agent.
+
+        When implementing, be sure to call `self._ready_event.set()` when the observer is ready to process events.
+
+        Args:
+            agent (RealtimeAgent): The realtime agent attached to the observer.
+        """
+        self._agent = agent
+        await self.initialize_session()
+        self._ready_event.set()
+
+        await self.run_loop()
+
+    @abstractmethod
+    async def run_loop(self) -> None:
+        """Run the loop if needed.
+
+        This method is called after the observer is ready to process events.
+        Events will be processed by the on_event method, this is just a hook for additional processing.
+        Use initialize_session to set up the session.
+        """
+        ...
+
+    @abstractmethod
+    async def initialize_session(self) -> None:
+        """Initialize the session for the observer."""
+        ...
+
+    async def wait_for_ready(self) -> None:
+        """Get the event that is set when the observer is ready."""
+        await self._ready_event.wait()
+
+    @abstractmethod
+    async def on_event(self, event: RealtimeEvent) -> None:
+        """Handle an event from the OpenAI Realtime API.
+
+        Args:
+            event (RealtimeServerEvent): The event from the OpenAI Realtime API.
+        """
+        ...
+
+    async def on_close(self) -> None:
+        """Handle close of RealtimeClient."""
+        ...