meshagent-livekit 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl

meshagent/livekit/agents/voice.py

@@ -1,22 +1,34 @@
 import logging
 import asyncio
 
-from meshagent.api import RoomMessage
+from meshagent.api import RoomMessage, ErrorResponse, JsonResponse, FileResponse, Requirement, Participant, JsonResponse, EmptyResponse, TextResponse
 from meshagent.api.room_server_client import RoomClient
 
+from meshagent.agents import ToolResponseAdapter
+from meshagent.tools import ToolContext, Toolkit
 from livekit.agents import Agent, AgentSession
+from livekit.agents.llm import RawFunctionTool, ToolError, function_tool
+
+from livekit.agents import BackgroundAudioPlayer, AudioConfig, BuiltinAudioClip
+
+from typing import Annotated
 from livekit.plugins import openai, silero
 #from livekit.plugins.turn_detector.multilingual import MultilingualModel
 import uuid
 import asyncio
 import logging
 
+import os
+
+import json 
+
+from typing import Any
 
 from livekit.plugins import openai
 
 from livekit.plugins import openai, silero
 from livekit import rtc
-from livekit.agents import Agent, AgentSession
+from livekit.agents import Agent, AgentSession, RunContext
 
 from typing import Optional
 
@@ -26,12 +38,39 @@ from meshagent.api.schema_util import merge, prompt_schema
 
 from meshagent.agents import SingleRoomAgent
 
+from livekit.plugins.turn_detector.multilingual import MultilingualModel
 
-logger = logging.getLogger("voice")
+import re
 
+logger = logging.getLogger("voice")
 
-from meshagent.agents.agent import AgentCallContext
 
+def _replace_non_matching(text: str, allowed_chars: str, replacement: str) -> str:
+    """
+    Replaces every character in `text` that does not match the given
+    `allowed_chars` regex set with `replacement`.
+    
+    Parameters:
+    -----------
+    text : str
+        The input string on which the replacement is to be done.
+    allowed_chars : str
+        A string defining the set of allowed characters (part of a character set).
+        For example, "a-zA-Z0-9" will keep only letters and digits.
+    replacement : str
+        The string to replace non-matching characters with.
+        
+    Returns:
+    --------
+    str
+        A new string where all characters not in `allowed_chars` are replaced.
+    """
+    # Build a regex that matches any character NOT in allowed_chars
+    pattern = rf"[^{allowed_chars}]"
+    return re.sub(pattern, replacement, text)
+
+def safe_tool_name(name: str):
+    return _replace_non_matching(name, "a-zA-Z0-9_-", "_")
 
 class VoiceConnection:
     def __init__(self, *, room: RoomClient, breakout_room: str):
@@ -69,10 +108,19 @@ class Voicebot(SingleRoomAgent):
         rules: Optional[list[str]] = None,
         auto_greet_prompt: Optional[str] = None,
         greeting: Optional[str] = None,
+        tool_adapter: ToolResponseAdapter = None,
+        toolkits: list[Toolkit] = None,
+        requires: list[Requirement] = None
     ):
+        if toolkits == None:
+            toolkits = []
+
+        self.toolkits = toolkits
+        
         if rules == None:
             rules = [ "You are a helpful assistant communicating through voice." ]
         
+        self.tool_adapter = tool_adapter
         self.auto_greet_prompt = auto_greet_prompt
         self.greeting = greeting
 
@@ -90,7 +138,8 @@ class Voicebot(SingleRoomAgent):
             name=name,
             description=description,
             title=title,
-            labels=labels
+            labels=labels,
+            requires=requires
         )
 
     async def start(self, *, room):
@@ -111,9 +160,16 @@ class Voicebot(SingleRoomAgent):
                 except Exception as e:
                     logger.error(f"{e}", exc_info=e)
 
-            task = asyncio.create_task(self.run_voice_agent(breakout_room=breakout_room))
-            task.add_done_callback(on_done)
-            
+            for participant in self.room.messaging.remote_participants:
+
+                if participant.id == message.from_participant_id:
+
+                    task = asyncio.create_task(self.run_voice_agent(participant=participant, breakout_room=breakout_room))
+                    task.add_done_callback(on_done)
+                    return
+                
+            logger.error(f"unable to find participant {message.from_participant_id}")
+                
     
     async def _wait_for_disconnect(self, room: rtc.Room):
         disconnected = asyncio.Future()
@@ -122,13 +178,106 @@ class Voicebot(SingleRoomAgent):
         room.on("disconnected", on_disconnected)
 
         logger.info("waiting for disconnection")
-        await disconnected      
+        await disconnected
+
+    async def make_function_tools(self, *, context: ToolContext):
+
+        toolkits = [
+            *await self.get_required_toolkits(context=context),
+            *self.toolkits
+        ]
+
+        tools = []
+
+        for toolkit in toolkits:
 
-    def create_agent(self):
-        return  Agent(
+            for tool in toolkit.tools:
+                
+                tools.append(self._make_function_tool(toolkits, context, tool.name, tool.description, tool.input_schema))
+
+        return tools
+
+    def _make_function_tool(
+        self, toolkits: list[Toolkit], context: ToolContext, name: str, description: str | None, input_schema: dict
+    ) -> RawFunctionTool:
+        
+        name = safe_tool_name(name)
+        async def _tool_called(raw_arguments: dict) -> Any:
+            try:
+
+                tool = None
+                for toolkit in toolkits:
+                    for t in toolkit.tools:
+                        if safe_tool_name(t.name) == name:
+                            tool = t
+
+                if tool is None:
+                    raise ToolError(
+                    f"Could not find tool {name}"
+                    )
+
+                try:
+                    logger.info(f"executing tool {name}: {raw_arguments}")
+                    tool_result = await tool.execute(context=context, **raw_arguments)
+                except Exception as e:
+                    logger.error(f"failed to call tool {tool.name}: {e}")
+                    return ToolError("f{e}")
+                if self.tool_adapter == None:
+
+                    if isinstance(tool_result, ErrorResponse):
+                        raise ToolError(tool_result.text)
+
+                    if isinstance(tool_result, JsonResponse):
+                        return json.dumps(tool_result.json)
+                    
+                    if isinstance(tool_result, TextResponse):
+                        return tool_result.text
+        
+                    if isinstance(tool_result, EmptyResponse):
+                        return "success"
+                    
+                    if tool_result == None:
+                        return "success"
+                    
+
+                    raise ToolError(
+                        f"Tool '{name}' returned an unexpected result {type(tool_result)}, attach a tool response adapter"
+                    )
+
+                else:
+                    
+                    text = await self.tool_adapter.to_plain_text(room=context.room, response=tool_result)
+                    if text == None:
+                        text = "success"
+                    return text
+
+            except Exception as e:
+                logger.error("unable to call tool", exc_info=e)
+                raise
+
+
+        return function_tool(
+            _tool_called,
+            raw_schema={"name": name, "description": description, "strict" : True, "parameters": input_schema},
+        )
+
+    async def create_agent(self, *, context: ToolContext, session: AgentSession):
+        
+        @function_tool
+        async def say(context: RunContext, text: str):
+            "says something out loud to the user"
+            logger.info(f"saying: {text}")
+            session.say(text)
+            return "success"
+
+        return Agent(
             instructions="\n".join(self.rules),
-            allow_interruptions=True
-        )     
+            allow_interruptions=True,
+            tools=[
+                *await self.make_function_tools(context=context),
+                say
+            ]
+        )
 
         # agent = Agent(
         #    instructions="""
@@ -144,32 +293,42 @@ class Voicebot(SingleRoomAgent):
     def create_session(self) -> AgentSession:
 
         session = AgentSession(
+            max_tool_steps=50,
             allow_interruptions=True,
             vad=silero.VAD.load(),
             stt=openai.STT(),
             tts=openai.TTS(voice="echo"),
-            llm=openai.realtime.RealtimeModel(
-                # it's necessary to turn off turn detection in the Realtime API in order to use
-                # LiveKit's turn detection model
-                voice="alloy",
-                turn_detection=None,
-                input_audio_transcription=None,
-
-            ),
+            llm=openai.LLM(),
+            #turn_detection=MultilingualModel(),
+
         )
         return session
     
     
-    async def run_voice_agent(self, *, breakout_room: str):
+    async def run_voice_agent(self, *, participant: Participant, breakout_room: str):
         
         async with VoiceConnection(room=self.room, breakout_room=breakout_room) as connection:
             
-
             logger.info("starting voice agent")
 
-            agent = self.create_agent()
             session = self.create_session()
 
+            agent = await self.create_agent(context=ToolContext(
+                room=self.room,
+                caller=self.room.local_participant,
+                on_behalf_of=participant
+            ), session=session)
+           
+            background_audio = BackgroundAudioPlayer(
+                thinking_sound=[
+                    #AudioConfig(
+                    #    os.path.dirname(os.path.abspath(__file__)) +"/sfx/thinking.mp3", volume=0.2),
+                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING, volume=0.3),
+                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING2, volume=0.4),
+                ],
+            )
+            await background_audio.start(room=connection.livekit_room, agent_session=session)
+
             await session.start(agent=agent, room=connection.livekit_room)
 
             if self.auto_greet_prompt != None:

meshagent/livekit/version.py

@@ -1 +1 @@
-__version__ = "0.0.19"
+__version__ = "0.0.21"

{meshagent_livekit-0.0.19.dist-info → meshagent_livekit-0.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshagent-livekit
-Version: 0.0.19
+Version: 0.0.21
 Summary: Livekit support for Meshagent
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://meshagent.com
@@ -13,13 +13,13 @@ Requires-Dist: pytest~=8.3.5
 Requires-Dist: pytest-asyncio~=0.26.0
 Requires-Dist: strip-markdown~=1.3
 Requires-Dist: livekit-api~=1.0.2
-Requires-Dist: livekit-agents~=1.0.11
-Requires-Dist: livekit-plugins-openai~=1.0.11
-Requires-Dist: livekit-plugins-cartesia~=1.0.11
-Requires-Dist: livekit-plugins-elevenlabs~=1.0.11
-Requires-Dist: livekit-plugins-playai~=1.0.11
-Requires-Dist: livekit-plugins-silero~=1.0.11
-Requires-Dist: livekit-plugins-turn-detector~=1.0.11
-Requires-Dist: meshagent-api~=0.0.19
-Requires-Dist: meshagent-tools~=0.0.19
+Requires-Dist: livekit-agents~=1.0.19
+Requires-Dist: livekit-plugins-openai~=1.0.19
+Requires-Dist: livekit-plugins-cartesia~=1.0.19
+Requires-Dist: livekit-plugins-elevenlabs~=1.0.19
+Requires-Dist: livekit-plugins-playai~=1.0.19
+Requires-Dist: livekit-plugins-silero~=1.0.19
+Requires-Dist: livekit-plugins-turn-detector~=1.0.19
+Requires-Dist: meshagent-api~=0.0.21
+Requires-Dist: meshagent-tools~=0.0.21
 Dynamic: license-file

{meshagent_livekit-0.0.19.dist-info → meshagent_livekit-0.0.21.dist-info}/RECORD

@@ -1,12 +1,12 @@
 meshagent/livekit/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
 meshagent/livekit/livekit_protocol.py,sha256=K9yP-qpxag5_7TXlKjFEx3cOJJJpYI_z6zGzFHoN1Hs,1421
 meshagent/livekit/livekit_protocol_test.py,sha256=n_ZQjt7n4u7TM7eENzH8L0tw8LvypS_JHF_PuJ2o6h4,2836
-meshagent/livekit/version.py,sha256=9OK8yo7gxBYvFxF5sp_Pg19ucyUGzDpUXRaYqvJmYCQ,22
+meshagent/livekit/version.py,sha256=9_nCx08vWHyj8RiwIYqwqah3T4SPB46e0jnaNSxxfDc,22
 meshagent/livekit/agents/transcriber.py,sha256=Dq1Ijx4gmA-0jQGM-f3w7X-JIZpkRCFDxWae9AOwz-k,12290
-meshagent/livekit/agents/voice.py,sha256=_kKMrOmxJ8324yZ1QdauuHlFb43P6xYbPeJzXH_nRwU,5410
+meshagent/livekit/agents/voice.py,sha256=uVpm-YeP1oXkg6gk1zFSwVEdp5XGQVimgs3DdNjEpbg,11230
 meshagent/livekit/tools/speech.py,sha256=UMhdHhTo04xdzHhvvCeTayT_YT86dzx4ZERRF18C0-o,10188
-meshagent_livekit-0.0.19.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
-meshagent_livekit-0.0.19.dist-info/METADATA,sha256=mp25p_3muvEtcy3lk5XcJsMbyCVmwZCyKz_GMO6gR5E,924
-meshagent_livekit-0.0.19.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-meshagent_livekit-0.0.19.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
-meshagent_livekit-0.0.19.dist-info/RECORD,,
+meshagent_livekit-0.0.21.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_livekit-0.0.21.dist-info/METADATA,sha256=HNq297tsUcj91no9YiCAUOlJZCGfQTMlpVmaUBeUF_8,924
+meshagent_livekit-0.0.21.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+meshagent_livekit-0.0.21.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_livekit-0.0.21.dist-info/RECORD,,

{meshagent_livekit-0.0.19.dist-info → meshagent_livekit-0.0.21.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any