meshagent-livekit 0.0.19__tar.gz → 0.0.21__tar.gz

{meshagent_livekit-0.0.19 → meshagent_livekit-0.0.21}/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.0.21]
+- Stability
+
+## [0.0.20]
+- Stability
+
 ## [0.0.19]
 - Stability
 

{meshagent_livekit-0.0.19/meshagent_livekit.egg-info → meshagent_livekit-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshagent-livekit
-Version: 0.0.19
+Version: 0.0.21
 Summary: Livekit support for Meshagent
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://meshagent.com
@@ -13,13 +13,13 @@ Requires-Dist: pytest~=8.3.5
 Requires-Dist: pytest-asyncio~=0.26.0
 Requires-Dist: strip-markdown~=1.3
 Requires-Dist: livekit-api~=1.0.2
-Requires-Dist: livekit-agents~=1.0.11
-Requires-Dist: livekit-plugins-openai~=1.0.11
-Requires-Dist: livekit-plugins-cartesia~=1.0.11
-Requires-Dist: livekit-plugins-elevenlabs~=1.0.11
-Requires-Dist: livekit-plugins-playai~=1.0.11
-Requires-Dist: livekit-plugins-silero~=1.0.11
-Requires-Dist: livekit-plugins-turn-detector~=1.0.11
-Requires-Dist: meshagent-api~=0.0.19
-Requires-Dist: meshagent-tools~=0.0.19
+Requires-Dist: livekit-agents~=1.0.19
+Requires-Dist: livekit-plugins-openai~=1.0.19
+Requires-Dist: livekit-plugins-cartesia~=1.0.19
+Requires-Dist: livekit-plugins-elevenlabs~=1.0.19
+Requires-Dist: livekit-plugins-playai~=1.0.19
+Requires-Dist: livekit-plugins-silero~=1.0.19
+Requires-Dist: livekit-plugins-turn-detector~=1.0.19
+Requires-Dist: meshagent-api~=0.0.21
+Requires-Dist: meshagent-tools~=0.0.21
 Dynamic: license-file

meshagent_livekit-0.0.21/meshagent/livekit/agents/voice.py

@@ -0,0 +1,346 @@
+import logging
+import asyncio
+
+from meshagent.api import RoomMessage, ErrorResponse, JsonResponse, FileResponse, Requirement, Participant, JsonResponse, EmptyResponse, TextResponse
+from meshagent.api.room_server_client import RoomClient
+
+from meshagent.agents import ToolResponseAdapter
+from meshagent.tools import ToolContext, Toolkit
+from livekit.agents import Agent, AgentSession
+from livekit.agents.llm import RawFunctionTool, ToolError, function_tool
+
+from livekit.agents import BackgroundAudioPlayer, AudioConfig, BuiltinAudioClip
+
+from typing import Annotated
+from livekit.plugins import openai, silero
+#from livekit.plugins.turn_detector.multilingual import MultilingualModel
+import uuid
+import asyncio
+import logging
+
+import os
+
+import json 
+
+from typing import Any
+
+from livekit.plugins import openai
+
+from livekit.plugins import openai, silero
+from livekit import rtc
+from livekit.agents import Agent, AgentSession, RunContext
+
+from typing import Optional
+
+from copy import deepcopy
+
+from meshagent.api.schema_util import merge, prompt_schema
+
+from meshagent.agents import SingleRoomAgent
+
+from livekit.plugins.turn_detector.multilingual import MultilingualModel
+
+import re
+
+logger = logging.getLogger("voice")
+
+
+def _replace_non_matching(text: str, allowed_chars: str, replacement: str) -> str:
+    """
+    Replaces every character in `text` that does not match the given
+    `allowed_chars` regex set with `replacement`.
+    
+    Parameters:
+    -----------
+    text : str
+        The input string on which the replacement is to be done.
+    allowed_chars : str
+        A string defining the set of allowed characters (part of a character set).
+        For example, "a-zA-Z0-9" will keep only letters and digits.
+    replacement : str
+        The string to replace non-matching characters with.
+        
+    Returns:
+    --------
+    str
+        A new string where all characters not in `allowed_chars` are replaced.
+    """
+    # Build a regex that matches any character NOT in allowed_chars
+    pattern = rf"[^{allowed_chars}]"
+    return re.sub(pattern, replacement, text)
+
+def safe_tool_name(name: str):
+    return _replace_non_matching(name, "a-zA-Z0-9_-", "_")
+
+class VoiceConnection:
+    def __init__(self, *, room: RoomClient, breakout_room: str):
+        self.room = room
+        self.breakout_room = breakout_room
+     
+    async def __aenter__(self):
+
+        client = self.room
+    
+        room_options = rtc.RoomOptions(auto_subscribe=True)
+
+        room = rtc.Room()
+
+        self.livekit_room = room
+
+        connection_info = await client.livekit.get_connection_info(breakout_room=self.breakout_room)
+        
+        await room.connect(url=connection_info.url, token=connection_info.token, options=room_options)
+
+        return self
+    
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.livekit_room.disconnect()
+
+class Voicebot(SingleRoomAgent):
+
+    def __init__(
+        self,
+        name: str,
+        input_schema: Optional[dict] = None, # the base schema, voice agent parameters will be added
+        title: Optional[str] = None,
+        description: Optional[str] = None,
+        labels: Optional[list[str]] = None,
+        rules: Optional[list[str]] = None,
+        auto_greet_prompt: Optional[str] = None,
+        greeting: Optional[str] = None,
+        tool_adapter: ToolResponseAdapter = None,
+        toolkits: list[Toolkit] = None,
+        requires: list[Requirement] = None
+    ):
+        if toolkits == None:
+            toolkits = []
+
+        self.toolkits = toolkits
+        
+        if rules == None:
+            rules = [ "You are a helpful assistant communicating through voice." ]
+        
+        self.tool_adapter = tool_adapter
+        self.auto_greet_prompt = auto_greet_prompt
+        self.greeting = greeting
+
+        self.rules = rules
+
+        if input_schema == None:
+            input_schema = None
+
+        input_schema = merge(
+            schema=input_schema,
+            additional_properties={
+        })
+        
+        super().__init__(
+            name=name,
+            description=description,
+            title=title,
+            labels=labels,
+            requires=requires
+        )
+
+    async def start(self, *, room):
+        await super().start(room=room)
+        await room.local_participant.set_attribute("supports_voice", True)
+        await room.messaging.enable()
+        room.messaging.on("message", self.on_message)
+    
+    def on_message(self, message: RoomMessage):
+        if message.type == "voice_call":
+            breakout_room = message.message["breakout_room"]
+            
+            logger.info(f"joining breakout room {breakout_room}")
+            
+            def on_done(task: asyncio.Task):
+                try:
+                    task.result()
+                except Exception as e:
+                    logger.error(f"{e}", exc_info=e)
+
+            for participant in self.room.messaging.remote_participants:
+
+                if participant.id == message.from_participant_id:
+
+                    task = asyncio.create_task(self.run_voice_agent(participant=participant, breakout_room=breakout_room))
+                    task.add_done_callback(on_done)
+                    return
+                
+            logger.error(f"unable to find participant {message.from_participant_id}")
+                
+    
+    async def _wait_for_disconnect(self, room: rtc.Room):
+        disconnected = asyncio.Future()
+        def on_disconnected(_):
+            disconnected.set_result(True)
+        room.on("disconnected", on_disconnected)
+
+        logger.info("waiting for disconnection")
+        await disconnected
+
+    async def make_function_tools(self, *, context: ToolContext):
+
+        toolkits = [
+            *await self.get_required_toolkits(context=context),
+            *self.toolkits
+        ]
+
+        tools = []
+
+        for toolkit in toolkits:
+
+            for tool in toolkit.tools:
+                
+                tools.append(self._make_function_tool(toolkits, context, tool.name, tool.description, tool.input_schema))
+
+        return tools
+
+    def _make_function_tool(
+        self, toolkits: list[Toolkit], context: ToolContext, name: str, description: str | None, input_schema: dict
+    ) -> RawFunctionTool:
+        
+        name = safe_tool_name(name)
+        async def _tool_called(raw_arguments: dict) -> Any:
+            try:
+
+                tool = None
+                for toolkit in toolkits:
+                    for t in toolkit.tools:
+                        if safe_tool_name(t.name) == name:
+                            tool = t
+
+                if tool is None:
+                    raise ToolError(
+                    f"Could not find tool {name}"
+                    )
+
+                try:
+                    logger.info(f"executing tool {name}: {raw_arguments}")
+                    tool_result = await tool.execute(context=context, **raw_arguments)
+                except Exception as e:
+                    logger.error(f"failed to call tool {tool.name}: {e}")
+                    return ToolError("f{e}")
+                if self.tool_adapter == None:
+
+                    if isinstance(tool_result, ErrorResponse):
+                        raise ToolError(tool_result.text)
+
+                    if isinstance(tool_result, JsonResponse):
+                        return json.dumps(tool_result.json)
+                    
+                    if isinstance(tool_result, TextResponse):
+                        return tool_result.text
+        
+                    if isinstance(tool_result, EmptyResponse):
+                        return "success"
+                    
+                    if tool_result == None:
+                        return "success"
+                    
+
+                    raise ToolError(
+                        f"Tool '{name}' returned an unexpected result {type(tool_result)}, attach a tool response adapter"
+                    )
+
+                else:
+                    
+                    text = await self.tool_adapter.to_plain_text(room=context.room, response=tool_result)
+                    if text == None:
+                        text = "success"
+                    return text
+
+            except Exception as e:
+                logger.error("unable to call tool", exc_info=e)
+                raise
+
+
+        return function_tool(
+            _tool_called,
+            raw_schema={"name": name, "description": description, "strict" : True, "parameters": input_schema},
+        )
+
+    async def create_agent(self, *, context: ToolContext, session: AgentSession):
+        
+        @function_tool
+        async def say(context: RunContext, text: str):
+            "says something out loud to the user"
+            logger.info(f"saying: {text}")
+            session.say(text)
+            return "success"
+
+        return Agent(
+            instructions="\n".join(self.rules),
+            allow_interruptions=True,
+            tools=[
+                *await self.make_function_tools(context=context),
+                say
+            ]
+        )
+
+        # agent = Agent(
+        #    instructions="""
+        #        You are a helpful assistant communicating through voice.
+        #    """,
+        #    stt=openai.STT(),
+        #    llm=openai.LLM(model="gpt-4o"),
+        #    tts=openai.TTS(),
+        #    vad=silero.VAD.load(),
+        #   allow_interruptions=True
+        #)      
+
+    def create_session(self) -> AgentSession:
+
+        session = AgentSession(
+            max_tool_steps=50,
+            allow_interruptions=True,
+            vad=silero.VAD.load(),
+            stt=openai.STT(),
+            tts=openai.TTS(voice="echo"),
+            llm=openai.LLM(),
+            #turn_detection=MultilingualModel(),
+
+        )
+        return session
+    
+    
+    async def run_voice_agent(self, *, participant: Participant, breakout_room: str):
+        
+        async with VoiceConnection(room=self.room, breakout_room=breakout_room) as connection:
+            
+            logger.info("starting voice agent")
+
+            session = self.create_session()
+
+            agent = await self.create_agent(context=ToolContext(
+                room=self.room,
+                caller=self.room.local_participant,
+                on_behalf_of=participant
+            ), session=session)
+           
+            background_audio = BackgroundAudioPlayer(
+                thinking_sound=[
+                    #AudioConfig(
+                    #    os.path.dirname(os.path.abspath(__file__)) +"/sfx/thinking.mp3", volume=0.2),
+                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING, volume=0.3),
+                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING2, volume=0.4),
+                ],
+            )
+            await background_audio.start(room=connection.livekit_room, agent_session=session)
+
+            await session.start(agent=agent, room=connection.livekit_room)
+
+            if self.auto_greet_prompt != None:
+                session.generate_reply(user_input=self.auto_greet_prompt)
+
+            if self.greeting != None:
+                session.say(self.greeting)
+            
+            logger.info("started voice agent")
+            await self._wait_for_disconnect(room=connection.livekit_room)
+            
+
+
+    
+

meshagent_livekit-0.0.21/meshagent/livekit/version.py

@@ -0,0 +1 @@
+__version__ = "0.0.21"

{meshagent_livekit-0.0.19 → meshagent_livekit-0.0.21/meshagent_livekit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshagent-livekit
-Version: 0.0.19
+Version: 0.0.21
 Summary: Livekit support for Meshagent
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://meshagent.com
@@ -13,13 +13,13 @@ Requires-Dist: pytest~=8.3.5
 Requires-Dist: pytest-asyncio~=0.26.0
 Requires-Dist: strip-markdown~=1.3
 Requires-Dist: livekit-api~=1.0.2
-Requires-Dist: livekit-agents~=1.0.11
-Requires-Dist: livekit-plugins-openai~=1.0.11
-Requires-Dist: livekit-plugins-cartesia~=1.0.11
-Requires-Dist: livekit-plugins-elevenlabs~=1.0.11
-Requires-Dist: livekit-plugins-playai~=1.0.11
-Requires-Dist: livekit-plugins-silero~=1.0.11
-Requires-Dist: livekit-plugins-turn-detector~=1.0.11
-Requires-Dist: meshagent-api~=0.0.19
-Requires-Dist: meshagent-tools~=0.0.19
+Requires-Dist: livekit-agents~=1.0.19
+Requires-Dist: livekit-plugins-openai~=1.0.19
+Requires-Dist: livekit-plugins-cartesia~=1.0.19
+Requires-Dist: livekit-plugins-elevenlabs~=1.0.19
+Requires-Dist: livekit-plugins-playai~=1.0.19
+Requires-Dist: livekit-plugins-silero~=1.0.19
+Requires-Dist: livekit-plugins-turn-detector~=1.0.19
+Requires-Dist: meshagent-api~=0.0.21
+Requires-Dist: meshagent-tools~=0.0.21
 Dynamic: license-file

meshagent_livekit-0.0.21/meshagent_livekit.egg-info/requires.txt

@@ -0,0 +1,13 @@
+pytest~=8.3.5
+pytest-asyncio~=0.26.0
+strip-markdown~=1.3
+livekit-api~=1.0.2
+livekit-agents~=1.0.19
+livekit-plugins-openai~=1.0.19
+livekit-plugins-cartesia~=1.0.19
+livekit-plugins-elevenlabs~=1.0.19
+livekit-plugins-playai~=1.0.19
+livekit-plugins-silero~=1.0.19
+livekit-plugins-turn-detector~=1.0.19
+meshagent-api~=0.0.21
+meshagent-tools~=0.0.21

{meshagent_livekit-0.0.19 → meshagent_livekit-0.0.21}/pyproject.toml

@@ -14,15 +14,15 @@ dependencies = [
   "pytest-asyncio~=0.26.0",
   "strip-markdown~=1.3",
   "livekit-api~=1.0.2",
-  "livekit-agents~=1.0.11",
-  "livekit-plugins-openai~=1.0.11",
-  "livekit-plugins-cartesia~=1.0.11",
-  "livekit-plugins-elevenlabs~=1.0.11",
-  "livekit-plugins-playai~=1.0.11",
-  "livekit-plugins-silero~=1.0.11",
-  "livekit-plugins-turn-detector~=1.0.11",
-  "meshagent-api~=0.0.19",
-  "meshagent-tools~=0.0.19"
+  "livekit-agents~=1.0.19",
+  "livekit-plugins-openai~=1.0.19",
+  "livekit-plugins-cartesia~=1.0.19",
+  "livekit-plugins-elevenlabs~=1.0.19",
+  "livekit-plugins-playai~=1.0.19",
+  "livekit-plugins-silero~=1.0.19",
+  "livekit-plugins-turn-detector~=1.0.19",
+  "meshagent-api~=0.0.21",
+  "meshagent-tools~=0.0.21"
 ]
 
 [project.urls]

meshagent_livekit-0.0.19/meshagent/livekit/agents/voice.py

@@ -1,187 +0,0 @@
-import logging
-import asyncio
-
-from meshagent.api import RoomMessage
-from meshagent.api.room_server_client import RoomClient
-
-from livekit.agents import Agent, AgentSession
-from livekit.plugins import openai, silero
-#from livekit.plugins.turn_detector.multilingual import MultilingualModel
-import uuid
-import asyncio
-import logging
-
-
-from livekit.plugins import openai
-
-from livekit.plugins import openai, silero
-from livekit import rtc
-from livekit.agents import Agent, AgentSession
-
-from typing import Optional
-
-from copy import deepcopy
-
-from meshagent.api.schema_util import merge, prompt_schema
-
-from meshagent.agents import SingleRoomAgent
-
-
-logger = logging.getLogger("voice")
-
-
-from meshagent.agents.agent import AgentCallContext
-
-
-class VoiceConnection:
-    def __init__(self, *, room: RoomClient, breakout_room: str):
-        self.room = room
-        self.breakout_room = breakout_room
-     
-    async def __aenter__(self):
-
-        client = self.room
-    
-        room_options = rtc.RoomOptions(auto_subscribe=True)
-
-        room = rtc.Room()
-
-        self.livekit_room = room
-
-        connection_info = await client.livekit.get_connection_info(breakout_room=self.breakout_room)
-        
-        await room.connect(url=connection_info.url, token=connection_info.token, options=room_options)
-
-        return self
-    
-    async def __aexit__(self, exc_type, exc, tb):
-        await self.livekit_room.disconnect()
-
-class Voicebot(SingleRoomAgent):
-
-    def __init__(
-        self,
-        name: str,
-        input_schema: Optional[dict] = None, # the base schema, voice agent parameters will be added
-        title: Optional[str] = None,
-        description: Optional[str] = None,
-        labels: Optional[list[str]] = None,
-        rules: Optional[list[str]] = None,
-        auto_greet_prompt: Optional[str] = None,
-        greeting: Optional[str] = None,
-    ):
-        if rules == None:
-            rules = [ "You are a helpful assistant communicating through voice." ]
-        
-        self.auto_greet_prompt = auto_greet_prompt
-        self.greeting = greeting
-
-        self.rules = rules
-
-        if input_schema == None:
-            input_schema = None
-
-        input_schema = merge(
-            schema=input_schema,
-            additional_properties={
-        })
-        
-        super().__init__(
-            name=name,
-            description=description,
-            title=title,
-            labels=labels
-        )
-
-    async def start(self, *, room):
-        await super().start(room=room)
-        await room.local_participant.set_attribute("supports_voice", True)
-        await room.messaging.enable()
-        room.messaging.on("message", self.on_message)
-    
-    def on_message(self, message: RoomMessage):
-        if message.type == "voice_call":
-            breakout_room = message.message["breakout_room"]
-            
-            logger.info(f"joining breakout room {breakout_room}")
-            
-            def on_done(task: asyncio.Task):
-                try:
-                    task.result()
-                except Exception as e:
-                    logger.error(f"{e}", exc_info=e)
-
-            task = asyncio.create_task(self.run_voice_agent(breakout_room=breakout_room))
-            task.add_done_callback(on_done)
-            
-    
-    async def _wait_for_disconnect(self, room: rtc.Room):
-        disconnected = asyncio.Future()
-        def on_disconnected(_):
-            disconnected.set_result(True)
-        room.on("disconnected", on_disconnected)
-
-        logger.info("waiting for disconnection")
-        await disconnected      
-
-    def create_agent(self):
-        return  Agent(
-            instructions="\n".join(self.rules),
-            allow_interruptions=True
-        )     
-
-        # agent = Agent(
-        #    instructions="""
-        #        You are a helpful assistant communicating through voice.
-        #    """,
-        #    stt=openai.STT(),
-        #    llm=openai.LLM(model="gpt-4o"),
-        #    tts=openai.TTS(),
-        #    vad=silero.VAD.load(),
-        #   allow_interruptions=True
-        #)      
-
-    def create_session(self) -> AgentSession:
-
-        session = AgentSession(
-            allow_interruptions=True,
-            vad=silero.VAD.load(),
-            stt=openai.STT(),
-            tts=openai.TTS(voice="echo"),
-            llm=openai.realtime.RealtimeModel(
-                # it's necessary to turn off turn detection in the Realtime API in order to use
-                # LiveKit's turn detection model
-                voice="alloy",
-                turn_detection=None,
-                input_audio_transcription=None,
-
-            ),
-        )
-        return session
-    
-    
-    async def run_voice_agent(self, *, breakout_room: str):
-        
-        async with VoiceConnection(room=self.room, breakout_room=breakout_room) as connection:
-            
-
-            logger.info("starting voice agent")
-
-            agent = self.create_agent()
-            session = self.create_session()
-
-            await session.start(agent=agent, room=connection.livekit_room)
-
-            if self.auto_greet_prompt != None:
-                session.generate_reply(user_input=self.auto_greet_prompt)
-
-            if self.greeting != None:
-                session.say(self.greeting)
-            
-            logger.info("started voice agent")
-            await self._wait_for_disconnect(room=connection.livekit_room)
-            
-
-
-    
-

meshagent_livekit-0.0.19/meshagent/livekit/version.py

@@ -1 +0,0 @@
-__version__ = "0.0.19"

meshagent_livekit-0.0.19/meshagent_livekit.egg-info/requires.txt

@@ -1,13 +0,0 @@
-pytest~=8.3.5
-pytest-asyncio~=0.26.0
-strip-markdown~=1.3
-livekit-api~=1.0.2
-livekit-agents~=1.0.11
-livekit-plugins-openai~=1.0.11
-livekit-plugins-cartesia~=1.0.11
-livekit-plugins-elevenlabs~=1.0.11
-livekit-plugins-playai~=1.0.11
-livekit-plugins-silero~=1.0.11
-livekit-plugins-turn-detector~=1.0.11
-meshagent-api~=0.0.19
-meshagent-tools~=0.0.19