beamlit 0.0.49rc96__py3-none-any.whl → 0.0.50rc98__py3-none-any.whl

beamlit/agents/chat.py

@@ -7,6 +7,7 @@ from beamlit.api.models import get_model
 from beamlit.authentication import get_authentication_headers, new_client
 from beamlit.common.settings import get_settings
 from beamlit.models import Model
+from .voice.openai import OpenAIVoiceReactAgent
 
 logger = getLogger(__name__)
 
@@ -57,7 +58,7 @@ def get_azure_ai_inference_chat_model(**kwargs):
 
 def get_azure_marketplace_chat_model(**kwargs):
     from langchain_openai import OpenAI  # type: ignore
-      
+
     return OpenAI(
         **kwargs
     )  # It seems to use a compatible endpoint, so we can use the classic OpenAI interface
@@ -156,6 +157,17 @@ def get_chat_model_full(name: str, agent_model: Union[Model, None] = None) -> Tu
         logger.warning("Model not found in agent model, defaulting to gpt-4o-mini")
         model = "gpt-4o-mini"
 
+    if provider == "openai" and "realtime" in model:
+        logger.info("Starting OpenAI Realtime Agent")
+        return (
+            OpenAIVoiceReactAgent(
+                url=get_base_url(name),
+                model=model,
+                headers=headers
+            ),
+            provider,
+            model
+        )
     kwargs = {
         "model": model,
         "base_url": get_base_url(name),

beamlit/agents/decorator.py

@@ -12,7 +12,7 @@ from beamlit.common.settings import init
 from beamlit.errors import UnexpectedStatus
 from beamlit.functions import get_functions
 from beamlit.models import Agent, AgentSpec, EnvironmentMetadata
-
+from .voice.openai import OpenAIVoiceReactAgent
 from .chat import get_chat_model_full
 
 
@@ -128,19 +128,22 @@ def agent(
                     f"        \"description\": \"{agent.spec.description}\",\n"
                     "    },\n"
                     "}")
-            memory = MemorySaver()
-            if len(functions) == 0:
-                raise ValueError("You can define this function in directory "
-                    f'"{settings.agent.functions_directory}". Here is a sample function you can use:\n\n'
-                    "from beamlit.functions import function\n\n"
-                    "@function()\n"
-                    "def hello_world(query: str):\n"
-                    "    return 'Hello, world!'\n")
-            try:
-                _agent = create_react_agent(chat_model, functions, checkpointer=memory)
-            except AttributeError: # special case for azure-marketplace where it uses the old OpenAI interface (no tools)
-                logger.warning("Using the old OpenAI interface for Azure Marketplace, no tools available")
-                _agent = create_react_agent(chat_model, [], checkpointer=memory)
+            if isinstance(chat_model, OpenAIVoiceReactAgent):
+                _agent = chat_model
+            else:
+                memory = MemorySaver()
+                if len(functions) == 0:
+                    raise ValueError("You can define this function in directory "
+                        f'"{settings.agent.functions_directory}". Here is a sample function you can use:\n\n'
+                        "from beamlit.functions import function\n\n"
+                        "@function()\n"
+                        "def hello_world(query: str):\n"
+                        "    return 'Hello, world!'\n")
+                try:
+                    _agent = create_react_agent(chat_model, functions, checkpointer=memory)
+                except AttributeError: # special case for azure-marketplace where it uses the old OpenAI interface (no tools)
+                    logger.warning("Using the old OpenAI interface for Azure Marketplace, no tools available")
+                    _agent = create_react_agent(chat_model, [], checkpointer=memory)
 
             settings.agent.agent = _agent
         else:

beamlit/agents/voice/openai.py

@@ -0,0 +1,257 @@
+import asyncio
+import json
+import websockets
+
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator, AsyncIterator, Any, Callable, Coroutine
+from .utils import amerge
+
+from langchain_core.tools import BaseTool
+from langchain_core._api import beta
+from langchain_core.utils import secret_from_env
+
+from pydantic import BaseModel, Field, SecretStr, PrivateAttr
+
+EVENTS_TO_IGNORE = {
+    "response.function_call_arguments.delta",
+    "rate_limits.updated",
+    "response.audio_transcript.delta",
+    "response.created",
+    "response.content_part.added",
+    "response.content_part.done",
+    "conversation.item.created",
+    "response.audio.done",
+    "session.created",
+    "session.updated",
+    "response.done",
+    "response.output_item.done",
+}
+
+
+@asynccontextmanager
+async def connect(*, headers: dict[str, Any], model: str, url: str) -> AsyncGenerator[
+    tuple[
+        Callable[[dict[str, Any] | str], Coroutine[Any, Any, None]],
+        AsyncIterator[dict[str, Any]],
+    ],
+    None,
+]:
+    """
+    async with connect(model="gpt-4o-realtime-preview-2024-10-01") as websocket:
+        await websocket.send("Hello, world!")
+        async for message in websocket:
+            print(message)
+    """
+    url += f"?model={model}"
+
+    websocket = await websockets.connect(url, extra_headers={**headers, "OpenAI-Beta": "realtime=v1"})
+
+    try:
+
+        async def send_event(event: dict[str, Any] | str) -> None:
+            formatted_event = json.dumps(event) if isinstance(event, dict) else event
+            await websocket.send(formatted_event)
+
+        async def event_stream() -> AsyncIterator[dict[str, Any]]:
+            async for raw_event in websocket:
+                yield json.loads(raw_event)
+
+        stream: AsyncIterator[dict[str, Any]] = event_stream()
+
+        yield send_event, stream
+    finally:
+        await websocket.close()
+
+
+class VoiceToolExecutor(BaseModel):
+    """
+    Can accept function calls and emits function call outputs to a stream.
+    """
+
+    tools_by_name: dict[str, BaseTool]
+    _trigger_future: asyncio.Future = PrivateAttr(default_factory=asyncio.Future)
+    _lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock)
+
+    async def _trigger_func(self) -> dict:  # returns a tool call
+        return await self._trigger_future
+
+    async def add_tool_call(self, tool_call: dict) -> None:
+        # lock to avoid simultaneous tool calls racing and missing
+        # _trigger_future being
+        async with self._lock:
+            if self._trigger_future.done():
+                # TODO: handle simultaneous tool calls better
+                raise ValueError("Tool call adding already in progress")
+
+            self._trigger_future.set_result(tool_call)
+
+    async def _create_tool_call_task(self, tool_call: dict) -> asyncio.Task[dict]:
+        tool = self.tools_by_name.get(tool_call["name"])
+        if tool is None:
+            # immediately yield error, do not add task
+            raise ValueError(
+                f"tool {tool_call['name']} not found. "
+                f"Must be one of {list(self.tools_by_name.keys())}"
+            )
+
+        # try to parse args
+        try:
+            args = json.loads(tool_call["arguments"])
+        except json.JSONDecodeError:
+            raise ValueError(
+                f"failed to parse arguments `{tool_call['arguments']}`. Must be valid JSON."
+            )
+
+        async def run_tool() -> dict:
+            result = await tool.ainvoke(args)
+            try:
+                result_str = json.dumps(result)
+            except TypeError:
+                # not json serializable, use str
+                result_str = str(result)
+            return {
+                "type": "conversation.item.create",
+                "item": {
+                    "id": tool_call["call_id"],
+                    "call_id": tool_call["call_id"],
+                    "type": "function_call_output",
+                    "output": result_str,
+                },
+            }
+
+        task = asyncio.create_task(run_tool())
+        return task
+
+    async def output_iterator(self) -> AsyncIterator[dict]:  # yield events
+        trigger_task = asyncio.create_task(self._trigger_func())
+        tasks = set([trigger_task])
+        while True:
+            done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+            for task in done:
+                tasks.remove(task)
+                if task == trigger_task:
+                    async with self._lock:
+                        self._trigger_future = asyncio.Future()
+                    trigger_task = asyncio.create_task(self._trigger_func())
+                    tasks.add(trigger_task)
+                    tool_call = task.result()
+                    try:
+                        new_task = await self._create_tool_call_task(tool_call)
+                        tasks.add(new_task)
+                    except ValueError as e:
+                        yield {
+                            "type": "conversation.item.create",
+                            "item": {
+                                "id": tool_call["call_id"],
+                                "call_id": tool_call["call_id"],
+                                "type": "function_call_output",
+                                "output": (f"Error: {str(e)}"),
+                            },
+                        }
+                else:
+                    yield task.result()
+
+
+@beta()
+class OpenAIVoiceReactAgent(BaseModel):
+    model: str
+    headers: dict[str, Any] = Field()
+    instructions: str | None = None
+    tools: list[BaseTool] | None = None
+    url: str = Field()
+
+    def bind_tools(self, tools: list[BaseTool]) -> None:
+        self.tools = tools
+
+    async def aconnect(
+        self,
+        input_stream: AsyncIterator[str],
+        send_output_chunk: Callable[[str], Coroutine[Any, Any, None]],
+    ) -> None:
+        """
+        Connect to the OpenAI API and send and receive messages.
+
+        input_stream: AsyncIterator[str]
+            Stream of input events to send to the model. Usually transports input_audio_buffer.append events from the microphone.
+        output: Callable[[str], None]
+            Callback to receive output events from the model. Usually sends response.audio.delta events to the speaker.
+
+        """
+        # formatted_tools: list[BaseTool] = [
+        #     tool if isinstance(tool, BaseTool) else tool_converter.wr(tool)  # type: ignore
+        #     for tool in self.tools or []
+        # ]
+        tools_by_name = {tool.name: tool for tool in self.tools}
+        tool_executor = VoiceToolExecutor(tools_by_name=tools_by_name)
+
+        async with connect(
+            model=self.model, headers=self.headers, url=self.url
+        ) as (
+            model_send,
+            model_receive_stream,
+        ):
+            # sent tools and instructions with initial chunk
+            tool_defs = [
+                {
+                    "type": "function",
+                    "name": tool.name,
+                    "description": tool.description,
+                    "parameters": {"type": "object", "properties": tool.args},
+                }
+                for tool in tools_by_name.values()
+            ]
+            await model_send(
+                {
+                    "type": "session.update",
+                    "session": {
+                        "instructions": self.instructions,
+                        "input_audio_transcription": {
+                            "model": "whisper-1",
+                        },
+                        "tools": tool_defs,
+                    },
+                }
+            )
+            async for stream_key, data_raw in amerge(
+                input_mic=input_stream,
+                output_speaker=model_receive_stream,
+                tool_outputs=tool_executor.output_iterator(),
+            ):
+                try:
+                    data = (
+                        json.loads(data_raw) if isinstance(data_raw, str) else data_raw
+                    )
+                except json.JSONDecodeError:
+                    print("error decoding data:", data_raw)
+                    continue
+
+                if stream_key == "input_mic":
+                    await model_send(data)
+                elif stream_key == "tool_outputs":
+                    print("tool output", data)
+                    await model_send(data)
+                    await model_send({"type": "response.create", "response": {}})
+                elif stream_key == "output_speaker":
+
+                    t = data["type"]
+                    if t == "response.audio.delta":
+                        await send_output_chunk(json.dumps(data))
+                    elif t == "input_audio_buffer.speech_started":
+                        print("interrupt")
+                        send_output_chunk(json.dumps(data))
+                    elif t == "error":
+                        print("error:", data)
+                    elif t == "response.function_call_arguments.done":
+                        print("tool call", data)
+                        await tool_executor.add_tool_call(data)
+                    elif t == "response.audio_transcript.done":
+                        print("model:", data["transcript"])
+                    elif t == "conversation.item.input_audio_transcription.completed":
+                        print("user:", data["transcript"])
+                    elif t in EVENTS_TO_IGNORE:
+                        pass
+                    else:
+                        print(t)
+
+
+__all__ = ["OpenAIVoiceReactAgent"]

beamlit/agents/voice/utils.py

@@ -0,0 +1,25 @@
+import asyncio
+from typing import AsyncIterator, TypeVar
+
+T = TypeVar("T")
+
+
+async def amerge(**streams: AsyncIterator[T]) -> AsyncIterator[tuple[str, T]]:
+    """Merge multiple streams into one stream."""
+    nexts: dict[asyncio.Task, str] = {
+        asyncio.create_task(anext(stream)): key for key, stream in streams.items()
+    }
+    while nexts:
+        done, _ = await asyncio.wait(nexts, return_when=asyncio.FIRST_COMPLETED)
+        for task in done:
+            key = nexts.pop(task)
+            stream = streams[key]
+            try:
+                yield key, task.result()
+                nexts[asyncio.create_task(anext(stream))] = key
+            except StopAsyncIteration:
+                pass
+            except Exception as e:
+                for task in nexts:
+                    task.cancel()
+                raise e

beamlit/deploy/deploy.py

@@ -191,7 +191,7 @@ COPY README.m[d] /beamlit/README.md
 COPY LICENS[E] /beamlit/LICENSE
 COPY {settings.server.directory} /beamlit/src
 
-RUN uv sync --no-cache
+RUN uv sync --no-cache --no-dev
 
 ENV PATH="/beamlit/.venv/bin:$PATH"
 

beamlit/serve/app.py

@@ -3,12 +3,13 @@ import importlib
 import os
 import sys
 import traceback
+import inspect
 from contextlib import asynccontextmanager
 from logging import getLogger
 from uuid import uuid4
 
 from asgi_correlation_id import CorrelationIdMiddleware
-from fastapi import FastAPI, Request, Response
+from fastapi import FastAPI, Request, Response, WebSocket
 from fastapi.responses import JSONResponse
 
 from beamlit.common import HTTPError, get_settings, init
@@ -38,7 +39,10 @@ logger.info(
     f"Running server with environment {settings.environment}"
     f" on {settings.server.host}:{settings.server.port}"
 )
-
+func_params = inspect.signature(func).parameters
+websocket_detected = False
+if "websocket" in func_params:
+    websocket_detected = True
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -56,48 +60,58 @@ app.add_middleware(AddProcessTimeHeader)
 app.add_middleware(AccessLogMiddleware)
 instrument_app(app)
 
-
 @app.get("/health")
 async def health():
     return {"status": "ok"}
 
+if websocket_detected:
+    @app.websocket("/ws")
+    async def websocket_endpoint(websocket: WebSocket):
+        await websocket.accept()
 
-@app.post("/")
-async def root(request: Request):
-    settings = get_settings()
-    logger = getLogger(__name__)
-    try:
         original_func = getattr(func, "__wrapped__", func)
         if asyncio.iscoroutinefunction(func) or asyncio.iscoroutinefunction(original_func):
-            response = await func(request)
+            await func(websocket)
         else:
-            response = func(request)
-
-        if isinstance(response, Response):
-            return response
-        if type(response) is str:
-            return Response(
-                content=response,
-                headers={"Content-Type": "text/plain"},
-                media_type="text/plain",
-                status_code=200,
-            )
-        return JSONResponse(status_code=200, content=response)
-    except ValueError as e:
-        content = {"error": str(e)}
-        if settings.environment == "development":
-            content["traceback"] = str(traceback.format_exc())
-        logger.error(str(traceback.format_exc()))
-        return JSONResponse(status_code=400, content=content)
-    except HTTPError as e:
-        content = {"error": e.message, "status_code": e.status_code}
-        if settings.environment == "development":
-            content["traceback"] = str(traceback.format_exc())
-        logger.error(f"{e.status_code} {str(traceback.format_exc())}")
-        return JSONResponse(status_code=e.status_code, content=content)
-    except Exception as e:
-        content = {"error": f"Internal server error, {e}"}
-        if settings.environment == "development":
-            content["traceback"] = str(traceback.format_exc())
-        logger.error(str(traceback.format_exc()))
-        return JSONResponse(status_code=500, content=content)
+            func(websocket)
+
+else:
+    @app.post("/")
+    async def root(request: Request):
+        settings = get_settings()
+        logger = getLogger(__name__)
+        try:
+            original_func = getattr(func, "__wrapped__", func)
+            if asyncio.iscoroutinefunction(func) or asyncio.iscoroutinefunction(original_func):
+                response = await func(request)
+            else:
+                response = func(request)
+
+            if isinstance(response, Response):
+                return response
+            if type(response) is str:
+                return Response(
+                    content=response,
+                    headers={"Content-Type": "text/plain"},
+                    media_type="text/plain",
+                    status_code=200,
+                )
+            return JSONResponse(status_code=200, content=response)
+        except ValueError as e:
+            content = {"error": str(e)}
+            if settings.environment == "development":
+                content["traceback"] = str(traceback.format_exc())
+            logger.error(str(traceback.format_exc()))
+            return JSONResponse(status_code=400, content=content)
+        except HTTPError as e:
+            content = {"error": e.message, "status_code": e.status_code}
+            if settings.environment == "development":
+                content["traceback"] = str(traceback.format_exc())
+            logger.error(f"{e.status_code} {str(traceback.format_exc())}")
+            return JSONResponse(status_code=e.status_code, content=content)
+        except Exception as e:
+            content = {"error": f"Internal server error, {e}"}
+            if settings.environment == "development":
+                content["traceback"] = str(traceback.format_exc())
+            logger.error(str(traceback.format_exc()))
+            return JSONResponse(status_code=500, content=content)

{beamlit-0.0.49rc96.dist-info → beamlit-0.0.50rc98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: beamlit
-Version: 0.0.49rc96
+Version: 0.0.50rc98
 Summary: Add your description here
 Author-email: cploujoux <ch.ploujoux@gmail.com>
 License-File: LICENSE
@@ -43,6 +43,7 @@ Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: pyyaml<6.1.0,>=6.0.2
 Requires-Dist: requests<2.33.0,>=2.32.3
+Requires-Dist: websockets>=14.1
 Description-Content-Type: text/markdown
 
 # beamlit

{beamlit-0.0.49rc96.dist-info → beamlit-0.0.50rc98.dist-info}/RECORD

@@ -6,9 +6,11 @@ beamlit/run.py,sha256=HtDYDjD7oVfQ8r3T5_t4qN5UDJOJfsQILi45Z21ArAg,1446
 beamlit/types.py,sha256=E1hhDh_zXfsSQ0NCt9-uw90_Mr5iIlsdfnfvxv5HarU,1005
 beamlit/agents/__init__.py,sha256=bWsFaXUbAps3IsL3Prti89m1s714vICXodbQi77h3vY,206
 beamlit/agents/chain.py,sha256=vfCjiFHuu02uTTGicxMlFzjyICQkIjpXrBGs-7uJEsg,2826
-beamlit/agents/chat.py,sha256=4R_mKeGvVXluQ5r-pnN3YaqhE3lFuetyos4zUBLnLnE,5510
-beamlit/agents/decorator.py,sha256=3-89dnk23l7M0Hxh---3pqVkiVDcAeFw9FLgBgSGDVQ,6358
+beamlit/agents/chat.py,sha256=wkzhRNWeKmNlaKZQVRLwdjTEtdOTiUnpj0fTeRe2ePk,5877
+beamlit/agents/decorator.py,sha256=5bEnVYEYTiFmCyQkFtN4XamrfTli6KYr98nFukPgOZE,6573
 beamlit/agents/thread.py,sha256=LN5Ss-uOf5_hdB0WV1dqpn-N-pDJB3C2hUvlCzdqtdk,519
+beamlit/agents/voice/openai.py,sha256=hmZQlAGQCQtR2MsXX1kC1DMUeiFctb-FiYqKDG7NGOo,9540
+beamlit/agents/voice/utils.py,sha256=tQidyM40Ewuy12wKqpvJLvfJgneQ0sZf50dqnerPGHg,836
 beamlit/api/__init__.py,sha256=zTSiG_ujSjAqWPyc435YXaX9XTlpMjiJWBbV-f-YtdA,45
 beamlit/api/accounts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 beamlit/api/accounts/delete_account.py,sha256=Mrbhuxl1IF1GDclSixIAvj3YHa7trlFg3JNmllqpBRA,3864
@@ -139,7 +141,7 @@ beamlit/common/settings.py,sha256=C7iZrLLvl6ap5VCVCYOoK20WdSFqacmd6HtaeZ7SF6s,37
 beamlit/common/slugify.py,sha256=nR29r37IdWS2i44ZC6ZsXRgqKPYmvMGtFQ7BuIQUTlc,90
 beamlit/common/utils.py,sha256=jouz5igBvT37Xn_e94-foCHyQczVim-UzVcoIF6RWJ4,657
 beamlit/deploy/__init__.py,sha256=GS7l7Jtm2yKs7iNLKcfjYO-rAhUzggQ3xiYSf3oxLBY,91
-beamlit/deploy/deploy.py,sha256=YqmaNJSa5faFpkUjBBX6ueAQxdmljdTdZBITMuaxY8U,10371
+beamlit/deploy/deploy.py,sha256=0snAN62iZGVYdH4Nh1H2-QTQKSN_-WpaEUCkwTo6USc,10380
 beamlit/deploy/format.py,sha256=U6UZEFAYLnGJJ7O2YmSdlUUFhnWNGAv6NZ-DW4KTgvI,2049
 beamlit/deploy/parser.py,sha256=Ga0poCZkoRnuTw082QnTcNGCBJncoRAnVsn8-1FsaJE,6907
 beamlit/functions/__init__.py,sha256=oejZ6GVd4-2kGKsOwRUq1u2AuSyrrn9kWIuH6WxMqhE,189
@@ -259,11 +261,11 @@ beamlit/models/websocket_channel.py,sha256=jg3vN7yS_oOIwGtndtIUr1LsyEA58RXLXahqS
 beamlit/models/workspace.py,sha256=nsck1BNzNoF_SalMqyVtkRoC_P-6-b4gzxKgKu3HwDw,5065
 beamlit/models/workspace_labels.py,sha256=WbnUY6eCTkUNdY7hhhSF-KQCl8fWFfkCf7hzCTiNp4A,1246
 beamlit/models/workspace_user.py,sha256=70CcifQWYbeWG7TDui4pblTzUe5sVK0AS19vNCzKE8g,3423
-beamlit/serve/app.py,sha256=ROS_tb9cO4GvOQKCwloyAzpYraTdIb3oG6sChXikeNw,3285
+beamlit/serve/app.py,sha256=BhEFQx8l2pv4eeVRGnvaC6el0tcgrzu-YiXpyL8oY6Q,3970
 beamlit/serve/middlewares/__init__.py,sha256=1dVmnOmhAQWvWktqHkKSIX-YoF6fmMU8xkUQuhg_rJU,148
 beamlit/serve/middlewares/accesslog.py,sha256=Mu4T4_9OvHybjA0ApzZFpgi2C8f3X1NbUk-76v634XM,631
 beamlit/serve/middlewares/processtime.py,sha256=lDAaIasZ4bwvN-HKHvZpaD9r-yrkVNZYx4abvbjbrCg,411
-beamlit-0.0.49rc96.dist-info/METADATA,sha256=6Eiz03TKilXs5O6NzHacJFjI-0Us6i8cZhClheKKT7E,3482
-beamlit-0.0.49rc96.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-beamlit-0.0.49rc96.dist-info/licenses/LICENSE,sha256=p5PNQvpvyDT_0aYBDgmV1fFI_vAD2aSV0wWG7VTgRis,1069
-beamlit-0.0.49rc96.dist-info/RECORD,,
+beamlit-0.0.50rc98.dist-info/METADATA,sha256=8tFF9JJsKQfaiy_El_dSMgkvAers9GIgsMsLcX13MNc,3514
+beamlit-0.0.50rc98.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+beamlit-0.0.50rc98.dist-info/licenses/LICENSE,sha256=p5PNQvpvyDT_0aYBDgmV1fFI_vAD2aSV0wWG7VTgRis,1069
+beamlit-0.0.50rc98.dist-info/RECORD,,