meshagent-livekit 0.0.15__py3-none-any.whl

meshagent/livekit/__init__.py

@@ -0,0 +1 @@
+from .version import __version__

meshagent/livekit/agents/transcriber.py

@@ -0,0 +1,298 @@
+import logging
+import asyncio
+
+from meshagent.api.schema_document import Element,Text
+from meshagent.api.room_server_client import RoomClient
+from meshagent.api.websocket_protocol import WebSocketClientProtocol
+
+
+import asyncio
+import os
+import logging
+import json
+
+from livekit import api
+
+from livekit.agents import stt, transcription, utils
+from livekit.plugins import openai, silero
+from livekit import rtc
+from livekit.rtc import TranscriptionSegment
+from livekit.agents import utils
+from livekit.agents import stt as speech_to_text
+
+from meshagent.api.runtime import RuntimeDocument
+
+from typing import Optional
+
+from meshagent.api.schema import MeshSchema
+
+from meshagent.api.schema import MeshSchema, ElementType, ChildProperty, ValueProperty
+
+from meshagent.agents.agent import AgentCallContext
+from meshagent.agents import TaskRunner
+
+logger = logging.getLogger("transcriber")
+
+
+transcription_schema = MeshSchema(
+    root_tag_name="transcript",
+    elements=[
+        ElementType(tag_name="transcript", description="a transcript", properties=[
+            ChildProperty(name="transcriptions", description="the transcript entries", child_tag_names=[ "speech" ])
+        ]),
+        ElementType(tag_name="speech", description="transcribed speech", properties=[
+            ValueProperty(name="text", description="the transcribed text", type="string"),
+            ValueProperty(name="startTime", description="the time of the start of this speech", type="number"),
+            ValueProperty(name="endTime", description="the time of th end of this speech", type="number"),
+            ValueProperty(name="participantId", description="the identity of the participant", type="string"),
+            ValueProperty(name="participantName", description="the name of the participant", type="string")
+        ])
+    ]
+)
+
+class Transcriber(TaskRunner):
+
+    def __init__(self, *, livekit_url: Optional[str] = None, livekit_api_key: Optional[str] = None, livekit_api_secret: Optional[str] = None, livekit_identity: Optional[str] = None):
+        super().__init__(
+            name="livekit.transcriber",
+            title="transcriber",
+            description="connects to a livekit room and transcribes the conversation",
+            input_schema={
+                "type" : "object",
+                "additionalProperties" : False,
+                "required" : [ "room_name", "path" ],
+                "properties" : {
+                    "room_name" : {
+                        "type": "string"
+                    },
+                    "path" : {
+                        "type" : "string"
+                    }
+                }
+            },
+            output_schema={
+                "type" : "object",
+                "additionalProperties" : False,
+                "required" : [],
+                "properties" : {
+                }
+            }
+        )
+        self._livekit_url = livekit_url
+        self._livekit_api_key = livekit_api_key
+        self._livekit_api_secret = livekit_api_secret
+        self._livekit_identity = livekit_identity
+
+    async def _transcribe_participant(self, doc: RuntimeDocument, room: rtc.Room, participant: rtc.RemoteParticipant,
+            stt_stream: stt.SpeechStream, stt_forwarder: transcription.STTSegmentsForwarder
+        ):
+            logger.info("transcribing participant %s", participant.sid)
+            """Forward the transcription to the client and log the transcript in the console"""
+            async for ev in stt_stream:
+                logger.info("event from participant %s %s", participant.sid, ev)
+                             
+                if ev.type == stt.SpeechEventType.FINAL_TRANSCRIPT:
+                    logger.info("transcript: %s", ev.alternatives[0].text)
+                    if len(ev.alternatives) > 0:
+                        alt = ev.alternatives[0]                                                
+                        doc.root.append_child(tag_name="speech", attributes={ "text":  alt.text, "startTime": alt.start_time, "endTime" : alt.end_time, "participantId" : participant.identity, "participantName" : participant.name })
+
+            logger.info("done forwarding %s", participant.sid)
+
+    def should_transcribe(self, p: rtc.Participant) -> bool:
+        # don't transcribe other agents
+        # todo: maybe have a better way to detect
+        return ".agent" not in p.identity
+    
+    async def _wait_for_disconnect(self, room: rtc.Room):
+        disconnected = asyncio.Future()
+        def on_disconnected(_):
+            disconnected.set_result(True)
+        room.on("disconnected", on_disconnected)
+
+        logger.info("waiting for disconnection")
+        await disconnected                 
+
+
+    async def ask(self, *, context: AgentCallContext, arguments: dict):
+        logger.info("Transcriber connecting to %s", arguments)
+        output_path = arguments["path"]
+        room_name = arguments["room_name"]
+
+        client = context.room        
+        doc = await client.sync.open(path=output_path)
+        try:
+
+            vad = silero.VAD.load()                
+            utils.http_context._new_session_ctx()              
+            
+            pending_tasks = list()
+            participantNames = dict[str,str]()
+
+            sst_provider = openai.STT()
+            #sst_provider = fal.WizperSTT()
+
+            room_options = rtc.RoomOptions(auto_subscribe=False)
+
+            room = rtc.Room()
+
+            url = self._livekit_url if self._livekit_url is not None else os.getenv('LIVEKIT_URL')
+            api_key = self._livekit_api_key if self._livekit_api_key is not None else os.getenv('LIVEKIT_API_KEY')
+            api_secret = self._livekit_api_secret if self._livekit_api_secret is not None else os.getenv('LIVEKIT_API_SECRET')
+            identity = self._livekit_identity if self._livekit_identity is not None else os.getenv('AGENT_IDENTITY')
+
+            token = api.AccessToken(api_key=api_key, api_secret=api_secret) \
+                .with_identity(identity) \
+                .with_name("Agent") \
+                .with_kind("agent") \
+                .with_grants(api.VideoGrants(
+                    can_update_own_metadata=True,
+                    room_join=True,
+                    room=room_name,
+                    agent=True
+                ))
+            
+            jwt = token.to_jwt()
+            
+            await room.connect(url=url, token=jwt, options=room_options)
+            
+
+            logger.info(
+                "connected to room: %s",
+                room_name
+            )
+        
+            audio_streams = list[rtc.AudioStream]()
+
+            async def transcribe_track(participant: rtc.RemoteParticipant, track: rtc.Track):
+                audio_stream = rtc.AudioStream(track)
+                stt_forwarder = transcription.STTSegmentsForwarder(
+                    room=room, participant=participant, track=track
+                )
+
+                audio_streams.append(audio_stream)
+                
+                stt = sst_provider
+                if not sst_provider.capabilities.streaming:                
+                    stt = speech_to_text.StreamAdapter(
+                        stt=stt,
+                        vad=vad,
+                    )
+                
+                stt_stream = stt.stream()
+
+                pending_tasks.append(asyncio.create_task(self._transcribe_participant(doc, room, participant, stt_stream, stt_forwarder)))
+
+                async for ev in audio_stream:
+                    stt_stream.push_frame(ev.frame)
+
+            def subscribe_if_needed(pub: rtc.RemoteTrackPublication):
+                if pub.kind == rtc.TrackKind.KIND_AUDIO:
+                    pub.set_subscribed(True)
+                
+            for p in room.remote_participants.values():
+                participantNames[p.identity] = p.name
+                if self.should_transcribe(p):
+                    for pub in p.track_publications.values():
+                        subscribe_if_needed(pub)
+            
+            first_parts = dict[str, rtc.Participant]()
+        
+            def on_transcript_event(segments: list[TranscriptionSegment], part: rtc.Participant | None, pub: rtc.TrackPublication | None = None) -> None:
+                nonlocal room
+                logger.info("Got transcription segment %s %s %s", segments, part, pub)
+                for segment in segments:
+                    if segment.id not in first_parts and part != None:
+                        first_parts[segment.id] = part
+
+                    if segment.final:
+                        if part == None and segment.id in first_parts:
+                            part = first_parts[segment.id]
+                            first_parts.pop(segment.id)
+                        
+                        if part != None:
+                            doc.root.append_child(tag_name="speech", attributes={ "text":  segment.text, "startTime": segment.start_time, "endTime" : segment.end_time, "participantId" : part.identity, "participantName" : part.name })
+                        else:
+                            logger.warning("transcription was missing participant information")
+
+            def on_participant_connected(p: rtc.RemoteParticipant):
+                participantNames[p.identity] = p.name
+
+            def on_track_published(pub: rtc.RemoteTrackPublication, p: rtc.RemoteParticipant):
+                if self.should_transcribe(p):
+                    subscribe_if_needed(pub)
+
+            subscriptions = dict()
+            
+            def on_track_unpublished(pub: rtc.RemoteTrackPublication, p: rtc.RemoteParticipant):
+                if pub in subscriptions:
+                    logger.info("track unpublished, stopping transcription")
+                    # todo: maybe could be more graceful
+                    subscriptions[pub].cancel()
+                    subscriptions.pop(pub)
+                
+            
+            def on_track_subscribed(
+                track: rtc.Track,
+                publication: rtc.TrackPublication,
+                participant: rtc.RemoteParticipant,
+            ):
+                if track.kind == rtc.TrackKind.KIND_AUDIO:
+                    logger.info("transcribing track %s", track.sid)
+                    track_task = asyncio.create_task(transcribe_track(participant, track))
+                    def on_transcription_done(t):
+                        try:
+                            t.result()
+                        except Exception as e:
+                            logger.error("Transcription failed", exc_info=e)
+
+
+                    track_task.add_done_callback(on_transcription_done)
+                    pending_tasks.append(track_task)
+                    subscriptions[publication] = track_task
+            
+
+            for p in room.remote_participants.values():
+                on_participant_connected(p)
+                            
+            room.on("participant_connected", on_participant_connected)
+
+            room.on("track_published", on_track_published)
+            room.on("track_unpublished", on_track_unpublished)
+            room.on("track_subscribed", on_track_subscribed)
+            room.on("transcription_received", on_transcript_event)
+
+            await self._wait_for_disconnect(room)
+                        
+            logger.info("waited for termination")
+            await room.disconnect()
+            
+            logger.info("closing audio streams")
+            
+            for stream in audio_streams:
+                await stream.aclose()
+
+    
+            logger.info("waiting for pending tasks")
+            gather_future = asyncio.gather(*pending_tasks)
+
+            gather_future.cancel()
+            try:
+                await gather_future
+            except Exception as e:
+                if isinstance(e, asyncio.CancelledError) == False:
+                    logger.warning("Did not shut down cleanly", exc_info=e)
+                pass
+
+            print("done")
+        except Exception as e:
+            logger.info("Transcription failed", exc_info=e)
+        finally:
+
+            await utils.http_context._close_http_ctx()
+            logger.info("Transcription done")
+
+            await asyncio.sleep(5)
+            await client.sync.close(path=output_path)
+            
+            return {}

meshagent/livekit/agents/voice.py

@@ -0,0 +1,187 @@
+import logging
+import asyncio
+
+from meshagent.api import RoomMessage
+from meshagent.api.room_server_client import RoomClient
+
+from livekit.agents import Agent, AgentSession
+from livekit.plugins import openai, silero
+#from livekit.plugins.turn_detector.multilingual import MultilingualModel
+import uuid
+import asyncio
+import logging
+
+
+from livekit.plugins import openai
+
+from livekit.plugins import openai, silero
+from livekit import rtc
+from livekit.agents import Agent, AgentSession
+
+from typing import Optional
+
+from copy import deepcopy
+
+from meshagent.api.schema_util import merge, prompt_schema
+
+from meshagent.agents import SingleRoomAgent
+
+
+logger = logging.getLogger("voice")
+
+
+from meshagent.agents.agent import AgentCallContext
+
+
+class VoiceConnection:
+    def __init__(self, *, room: RoomClient, breakout_room: str):
+        self.room = room
+        self.breakout_room = breakout_room
+     
+    async def __aenter__(self):
+
+        client = self.room
+    
+        room_options = rtc.RoomOptions(auto_subscribe=True)
+
+        room = rtc.Room()
+
+        self.livekit_room = room
+
+        connection_info = await client.livekit.get_connection_info(breakout_room=self.breakout_room)
+        
+        await room.connect(url=connection_info.url, token=connection_info.token, options=room_options)
+
+        return self
+    
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.livekit_room.disconnect()
+
+class Voice(SingleRoomAgent):
+
+    def __init__(
+        self,
+        name: str,
+        input_schema: Optional[dict] = None, # the base schema, voice agent parameters will be added
+        title: Optional[str] = None,
+        description: Optional[str] = None,
+        labels: Optional[list[str]] = None,
+        rules: Optional[list[str]] = None,
+        auto_greet_prompt: Optional[str] = None,
+        greeting: Optional[str] = None,
+    ):
+        if rules == None:
+            rules = [ "You are a helpful assistant communicating through voice." ]
+        
+        self.auto_greet_prompt = auto_greet_prompt
+        self.greeting = greeting
+
+        self.rules = rules
+
+        if input_schema == None:
+            input_schema = None
+
+        input_schema = merge(
+            schema=input_schema,
+            additional_properties={
+        })
+        
+        super().__init__(
+            name=name,
+            description=description,
+            title=title,
+            labels=labels
+        )
+
+    async def start(self, *, room):
+        await super().start(room=room)
+        await room.local_participant.set_attribute("supports_voice", True)
+        await room.messaging.enable()
+        room.messaging.on("message", self.on_message)
+    
+    def on_message(self, message: RoomMessage):
+        if message.type == "voice_call":
+            breakout_room = message.message["breakout_room"]
+            
+            logger.info(f"joining breakout room {breakout_room}")
+            
+            def on_done(task: asyncio.Task):
+                try:
+                    task.result()
+                except Exception as e:
+                    logger.error(f"{e}", exc_info=e)
+
+            task = asyncio.create_task(self.run_voice_agent(breakout_room=breakout_room))
+            task.add_done_callback(on_done)
+            
+    
+    async def _wait_for_disconnect(self, room: rtc.Room):
+        disconnected = asyncio.Future()
+        def on_disconnected(_):
+            disconnected.set_result(True)
+        room.on("disconnected", on_disconnected)
+
+        logger.info("waiting for disconnection")
+        await disconnected      
+
+    def create_agent(self):
+        return  Agent(
+            instructions="\n".join(self.rules),
+            allow_interruptions=True
+        )     
+
+        # agent = Agent(
+        #    instructions="""
+        #        You are a helpful assistant communicating through voice.
+        #    """,
+        #    stt=openai.STT(),
+        #    llm=openai.LLM(model="gpt-4o"),
+        #    tts=openai.TTS(),
+        #    vad=silero.VAD.load(),
+        #   allow_interruptions=True
+        #)      
+
+    def create_session(self) -> AgentSession:
+
+        session = AgentSession(
+            allow_interruptions=True,
+            vad=silero.VAD.load(),
+            stt=openai.STT(),
+            tts=openai.TTS(voice="echo"),
+            llm=openai.realtime.RealtimeModel(
+                # it's necessary to turn off turn detection in the Realtime API in order to use
+                # LiveKit's turn detection model
+                voice="alloy",
+                turn_detection=None,
+                input_audio_transcription=None,
+
+            ),
+        )
+        return session
+    
+    
+    async def run_voice_agent(self, *, breakout_room: str):
+        
+        async with VoiceConnection(room=self.room, breakout_room=breakout_room) as connection:
+            
+
+            logger.info("starting voice agent")
+
+            agent = self.create_agent()
+            session = self.create_session()
+
+            await session.start(agent=agent, room=connection.livekit_room)
+
+            if self.auto_greet_prompt != None:
+                session.generate_reply(user_input=self.auto_greet_prompt)
+
+            if self.greeting != None:
+                session.say(self.greeting)
+            
+            logger.info("started voice agent")
+            await self._wait_for_disconnect(room=connection.livekit_room)
+            
+
+
+    
+

meshagent/livekit/livekit_protocol.py

@@ -0,0 +1,48 @@
+from meshagent.api.protocol import Protocol
+
+from livekit import rtc
+import logging
+
+logger = logging.getLogger("protocol.livekit")
+
+
+class LivekitProtocol(Protocol):
+    def __init__(self, room: rtc.Room, remote: rtc.RemoteParticipant, topic: str):
+        super().__init__()
+        self.room = room
+        self.local = room.local_participant
+        self.remote = remote
+        self.topic = topic
+
+
+    async def __aenter__(self):
+        self.room.on("data_received", self._on_data_packet)
+        return await super().__aenter__()
+    
+    async def __aexit__(self, exc_type, exc, tb):
+        self.room.off("data_received", self._on_data_packet)
+    
+        return await super().__aexit__(exc_type, exc, tb)
+
+    async def send_packet(self, data:bytes) -> None:
+
+        logger.info("sending data packet %s  %s to %s", self.topic, self.remote.identity, self.room.remote_participants[self.remote.identity].sid)
+
+        await self.local.publish_data(
+            payload=data,
+            topic = self.topic,
+            reliable = True,
+            destination_identities = [ self.remote.identity ],
+        )
+
+    def _on_data_packet(self, evt: rtc.DataPacket):
+        
+        if self.remote != evt.participant:
+            return
+
+        logger.info("received data packet %s from %s", evt.topic, evt.participant.identity)
+
+        if evt.topic == self.topic:
+            self.receive_packet(evt.data)
+
+     

meshagent/livekit/livekit_protocol_test.py

@@ -0,0 +1,106 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import logging
+import pytest
+import os
+
+from livekit import api
+from livekit import rtc
+
+import livekit_protocol
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+@pytest.mark.asyncio
+async def test_protocol():
+
+    url = os.getenv('LIVEKIT_URL')
+    api_key = os.getenv('LIVEKIT_API_KEY')
+    api_secret = os.getenv('LIVEKIT_API_SECRET')
+
+
+    token1 = api.AccessToken(api_key=api_key, api_secret=api_secret) \
+        .with_identity('core:user.test.agent-send') \
+        .with_name("Agent") \
+        .with_kind("agent") \
+        .with_grants(api.VideoGrants(
+            can_update_own_metadata=True,
+            room_join=True,
+            room="test-process",
+            agent=True
+        ))
+    
+    jwt1 = token1.to_jwt()
+
+
+    token2 = api.AccessToken(api_key=api_key, api_secret=api_secret) \
+        .with_identity('core:user.test.agent-recv') \
+        .with_name("Agent") \
+        .with_kind("agent") \
+        .with_grants(api.VideoGrants(
+            can_update_own_metadata=True,
+            room_join=True,
+            room="test-process",
+            agent=True
+        ))
+    
+    jwt2 = token2.to_jwt()       
+
+    room1 = rtc.Room()  
+    await room1.connect(url=url, token=jwt1)
+    
+    room2 = rtc.Room()  
+    await room2.connect(url=url, token=jwt2)
+                  
+    topic = "test_topic"
+
+    while True:
+        await asyncio.sleep(0.1)
+
+        if room2.local_participant.identity in room1.remote_participants and room1.local_participant.identity in room2.remote_participants:
+            break
+
+    async with livekit_protocol.LivekitProtocol(room = room1, remote = room1.remote_participants[room2.local_participant.identity], topic=topic) as proto1:
+        async  with livekit_protocol.LivekitProtocol(room = room2, remote = room2.remote_participants[room1.local_participant.identity], topic=topic) as proto2:
+
+            test_data_builder = bytearray()
+            for i in range(1024*1024):
+                test_data_builder.append(i % 255)
+
+            test_data = bytes(test_data_builder)
+
+            done = asyncio.Future[bool]()
+
+            matches = 0
+            async def test_fn(protocol, id: int, type:str, data:bytes):
+                nonlocal matches
+                logger.info("Message received")
+                if test_data != data:
+                    raise "data isn't equal"
+                matches+=1
+
+                if matches == 2:
+                    done.set_result(True)
+            
+            proto2.register_handler("test", test_fn)
+
+            await asyncio.sleep(1)
+
+            await proto1.send("test", test_data)
+            await proto1.send("test", test_data)
+
+            await done
+
+
+    await room2.disconnect()
+    await room1.disconnect()
+
+
+    
+
+
+    

meshagent/livekit/tools/speech.py

@@ -0,0 +1,260 @@
+import uuid
+from livekit import rtc
+from livekit.agents.tts import TTS
+from typing import Optional
+
+from ..agents.voice import VoiceConnection
+from meshagent.tools.toolkit import Toolkit, Tool, FileResponse, ToolContext, TextResponse
+
+from livekit.plugins import elevenlabs
+from livekit.plugins import cartesia
+from livekit.plugins import openai
+from livekit.plugins import playai
+
+class SpeechTools(Toolkit):
+    def __init__(self):
+        super().__init__(
+        name="meshagent.speech",
+        title="voice",
+        description="speech to text tools",    
+        tools=[
+            ElevenTextToSpeech(),
+            CartesiaTextToSpeech(),
+            OpenAITextToSpeech(),
+            PlayHTTextToSpeech(),
+        ])
+
+
+async def synthesize(tts: TTS, text:str):
+
+    frames = list[rtc.AudioFrame]()
+    stream = tts.synthesize(text=text)
+    try:
+        async for chunk in stream:
+            frame : rtc.AudioFrame = chunk.frame
+            frames.append(frame)
+
+        merged = rtc.combine_audio_frames(frames)
+        return FileResponse(data=merged.to_wav_bytes(), name= str(uuid.uuid4())+".wav", mime_type="audio/wav")
+    finally:
+        await stream.aclose()
+
+
+class PlayHTTextToSpeech(Tool):
+    def __init__(self):
+        super().__init__(
+            name="playht_text_to_speech",
+            title="PlayHT text to speech",
+            description="generate an audio file, converting text to speech",
+            input_schema={
+                "type" : "object",
+                "properties" : {
+                    "input_text" : {
+                        "type" : "string",
+                        "description" : "the text to convert to speech",
+                    },
+                    "model" : {
+                        "type": "string",
+                        "description" : "(default: PlayDialog)",
+                        "enum" : [ "Play3.0-mini", "PlayDialog", "PlayHT2.0-turbo" ]
+                    },
+                    "sample_rate" : {
+                        "type" : "number",
+                        "description" : "(default: 48000)",
+                        "enum" : [ 48000, 24000 ]
+                    }
+                },
+                "required": ["input_text","model","sample_rate"],
+                "additionalProperties" : False,   
+        })
+
+  
+    
+    async def execute(self, *, context: ToolContext, input_text: str, sample_rate: int, model: str):
+        tts = playai.TTS(
+            model=model,
+            sample_rate=sample_rate,
+        )
+
+        return await synthesize(tts, input_text)
+
+class ElevenTextToSpeech(Tool):
+    def __init__(self):
+        super().__init__(
+            name="eleven_labs_text_to_speech",
+            title="ElevenLabs text to speech",
+            description="generate an audio file, converting text to speech",
+            input_schema={
+                "type" : "object",
+                "properties" : {
+                    "input_text" : {
+                        "type" : "string",
+                        "description" : "the text to convert to speech",
+                    },
+                    "voice_id" : {
+                        "type" : "string",
+                        "description" : "the id of a voice to use (default: EXAVITQu4vr4xnSDxMaL)",
+                    },
+                    "voice_name" : {
+                        "type" : "string",
+                        "description" : "the name of the voice to use (optional)",
+                    },
+                    "voice_category" : {
+                        "type" : "string",
+                        "description" : "the category of the voice to use (optional, default: premade)",
+                        "enum" : [
+                            "generated", "cloned", "premade", "professional", "famous", "high_quality"
+                        ]
+                    },
+                    "model" : {
+                        "type": "string",
+                        "description" : "(default: eleven_flash_v2_5)",
+                        "enum" : [ "eleven_multilingual_v2", "eleven_flash_v2_5", "eleven_flash_v2", "eleven_multilingual_sts_v2", "eleven_english_sts_v2" ]
+                    },
+                    "encoding" : {
+                        "type" : "string",
+                        "description" : "(default: pcm_44100)",
+                        "enum" : [ "pcm_44100", "mp3_22050_32" ]
+                    }
+                },
+                "required": ["input_text","voice_id","model","encoding", "voice_name", "voice_category"],
+                "additionalProperties" : False,   
+        })
+
+  
+    
+    async def execute(self, *, context: ToolContext, input_text: str, voice_id: str, voice_name:str, voice_category:str, model: str, encoding: str):
+        tts = elevenlabs.TTS(
+            model_id=model,
+            encoding=encoding,
+            voice=elevenlabs.Voice(id=voice_id, name=voice_name, category=voice_category)
+        )
+
+        return await synthesize(tts, input_text)
+
+
+class CartesiaTextToSpeech(Tool):
+    def __init__(self):
+        super().__init__(
+            name="cartesia_text_to_speech",
+            title="Cartesia text to speech",
+            description="generate an audio file, converting text to speech",
+            input_schema={
+                "type" : "object",
+                "properties" : {
+                    "input_text" : {
+                        "type" : "string",
+                        "description" : "the text to convert to speech",
+                    },
+                    "voice" : {
+                        "type" : "string",
+                        "description" : "the id of a voice to use (default: c2ac25f9-ecc4-4f56-9095-651354df60c0)"
+                    },
+                    "model" : {
+                        "type": "string",
+                        "description" : "(default: sonic-english)",
+                        "enum" : [ "sonic", "sonic-preview", "sonic-2024-12-12", "sonic-2024-10-19", "sonic-english", "sonic-multilingual" ]
+                    },
+                    "speed" : {
+                        "type" : "string",
+                        "description" : "(default: normal)",
+                        "enum" : ["fastest", "fast", "normal", "slow", "slowest" ]
+                    },
+                    "encoding" : {
+                        "type" : "string",
+                        "description" : "(default: pcm_s16le)",
+                        "enum" : [ "pcm_s16le" ]
+                    },
+                    "emotion" : {
+                        "type" : "array",
+                        "items" : {
+                            "type": "string",
+                            "enum" : [ 
+                                "anger:lowest", 
+                                "positivity:lowest",
+                                "surprise:lowest",
+                                "sadness:lowest",
+                                "curiosity:lowest" 
+                                "anger:low", 
+                                "positivity:low",
+                                "surprise:low",
+                                "sadness:low",
+                                "curiosity:low" 
+                                "anger:medium", 
+                                "positivity",
+                                "surprise",
+                                "sadness",
+                                "curiosity" 
+                                "anger:high", 
+                                "positivity:high",
+                                "surprise:high",
+                                "sadness:high",
+                                "curiosity:high",
+                                "anger:highest", 
+                                "positivity:highest",
+                                "surprise:highest",
+                                "sadness:highest",
+                                "curiosity:highest" 
+
+                            ]
+                        }
+                    }
+                },
+                "required": ["input_text", "voice","speed","emotion","encoding","model"],
+                "additionalProperties" : False,   
+        })
+    
+    async def execute(self, *, context: ToolContext, input_text: str, voice: str, model: str, speed: str, emotion: list, encoding: str):
+        tts = cartesia.TTS(
+            encoding=encoding,
+            model=model,
+            voice=voice,
+            emotion=emotion,
+            speed=speed
+        )
+
+        return await synthesize(tts, input_text)  
+
+
+class OpenAITextToSpeech(Tool):
+    def __init__(self):
+        super().__init__(
+            name="openai_text_to_speech",
+            title="OpenAI text to speech",
+            description="generate an audio file, converting text to speech",
+            input_schema={
+                "type" : "object",
+                "properties" : {
+                    "input_text" : {
+                        "type" : "string",
+                        "description" : "the text to convert to speech",
+                    },
+                    "voice" : {
+                        "type" : "string",
+                        "description" : "the id of a voice to use (default: alloy)",
+                        "enum" : [
+                            "alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"
+                        ]
+                    },
+                    "model" : {
+                        "type": "string",
+                        "description" : "(default: tts-1)",
+                        "enum" : [ "tts-1", "tts-1-hd" ]
+                    },
+                    "speed" : {
+                        "type" : "number",
+                        "description" : "(default: 1.0)",
+                    },
+                },
+                "required": ["input_text", "voice", "model", "speed"],
+                "additionalProperties" : False,   
+        })
+    
+    async def execute(self, *, context: ToolContext, input_text: str, voice: str, model: str, speed: float):
+        tts = openai.TTS(
+            model=model,
+            voice=voice,
+            speed=speed
+        )
+
+        return await synthesize(tts, input_text)

meshagent/livekit/version.py

@@ -0,0 +1 @@
+__version__ = "0.0.15"

meshagent_livekit-0.0.15.dist-info/METADATA

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: meshagent-livekit
+Version: 0.0.15
+Summary: Livekit support for Meshagent
+License-Expression: Apache-2.0
+Project-URL: Documentation, https://meshagent.com
+Project-URL: Website, https://meshagent.com
+Project-URL: Source, https://github.com/meshagent
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest~=8.3.5
+Requires-Dist: pytest-asyncio~=0.26.0
+Requires-Dist: strip-markdown~=1.3
+Requires-Dist: livekit-api~=1.0.2
+Requires-Dist: livekit-agents~=1.0.11
+Requires-Dist: livekit-plugins-openai~=1.0.11
+Requires-Dist: livekit-plugins-cartesia~=1.0.11
+Requires-Dist: livekit-plugins-elevenlabs~=1.0.11
+Requires-Dist: livekit-plugins-playai~=1.0.11
+Requires-Dist: livekit-plugins-silero~=1.0.11
+Requires-Dist: livekit-plugins-turn-detector~=1.0.11
+Requires-Dist: meshagent-api~=0.0.15
+Requires-Dist: meshagent-tools~=0.0.15
+Dynamic: license-file

meshagent_livekit-0.0.15.dist-info/RECORD

@@ -0,0 +1,12 @@
+meshagent/livekit/__init__.py,sha256=8zLGg-DfQhnDl2Ky0n-zXpN-8e-g7iR0AcaI4l4Vvpk,32
+meshagent/livekit/livekit_protocol.py,sha256=K9yP-qpxag5_7TXlKjFEx3cOJJJpYI_z6zGzFHoN1Hs,1421
+meshagent/livekit/livekit_protocol_test.py,sha256=n_ZQjt7n4u7TM7eENzH8L0tw8LvypS_JHF_PuJ2o6h4,2836
+meshagent/livekit/version.py,sha256=OpICqW3gacySIEm2aFlBiaIhB7efvBWC8Mt6amfme9M,22
+meshagent/livekit/agents/transcriber.py,sha256=Dq1Ijx4gmA-0jQGM-f3w7X-JIZpkRCFDxWae9AOwz-k,12290
+meshagent/livekit/agents/voice.py,sha256=g1dLF33qGQp6vex-nNhiKMpYX119u1Nsqx96kvYG0K8,5407
+meshagent/livekit/tools/speech.py,sha256=UMhdHhTo04xdzHhvvCeTayT_YT86dzx4ZERRF18C0-o,10188
+meshagent_livekit-0.0.15.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_livekit-0.0.15.dist-info/METADATA,sha256=NHRkl5SOE4cm6g7W3GKKESPDNWozmYjn4baUtHiLVmY,924
+meshagent_livekit-0.0.15.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+meshagent_livekit-0.0.15.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_livekit-0.0.15.dist-info/RECORD,,

meshagent_livekit-0.0.15.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (79.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

meshagent_livekit-0.0.15.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Don't include
+the brackets!)  The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

meshagent_livekit-0.0.15.dist-info/top_level.txt

@@ -0,0 +1 @@
+meshagent