meshagent-livekit 0.8.3__py3-none-any.whl → 0.20.1__py3-none-any.whl

meshagent/livekit/agents/meeting_transcriber.py

@@ -3,28 +3,28 @@ import logging
 
 from typing import Optional
 
-from datetime import datetime, timezone
 from livekit import rtc
 from livekit.agents import (
-    Agent,
     AgentSession,
     RoomInputOptions,
     RoomIO,
     RoomOutputOptions,
-    StopResponse,
-    llm,
     utils,
 )
 from livekit.agents.stt import STT
 
+from meshagent.api import RoomException
+
 from meshagent.openai.proxy import get_client
 from livekit.plugins import openai, silero
-from meshagent.api import MeshDocument, SchemaRegistration, SchemaRegistry
+from meshagent.api import SchemaRegistration, SchemaRegistry
 from meshagent.agents import SingleRoomAgent
 from meshagent.tools import RemoteToolkit, ToolContext, Tool
 from meshagent.api.room_server_client import Requirement
 from meshagent.livekit.agents.voice import VoiceConnection
 from meshagent.agents.schemas.transcript import transcript_schema
+from meshagent.livekit.agents.transcript_logger import Transcriber
+
 
 logger = logging.getLogger("meeting_transcriber")
 
@@ -127,14 +127,14 @@ class MeetingTranscriber(SingleRoomAgent):
                 async with VoiceConnection(
                     room=self.room, breakout_room=breakout_room
                 ) as conn:
-                    doc = await self.room.sync.open(path=path, create=True)
-
                     stt = self.stt
                     if stt is None:
                         openai_client = get_client(room=self.room)
                         stt = openai.STT(client=openai_client)
 
-                    transcriber = MultiUserTranscriber(conn, doc, self._vad, stt)
+                    transcriber = MultiUserTranscriber(
+                        conn, path, self.room, self._vad, stt
+                    )
                     transcriber.start()
 
                     for participant in conn.livekit_room.remote_participants.values():
@@ -147,8 +147,6 @@ class MeetingTranscriber(SingleRoomAgent):
                         f"transcribing.{breakout_room}", False
                     )
 
-                    await self.room.sync.close(path=path)
-
                     await transcriber.aclose()
             except Exception as ex:
                 logger.error(f"error during transcription {ex}", exc_info=ex)
@@ -193,35 +191,11 @@ class TranscriptRegistry(SchemaRegistry):
         )
 
 
-class Transcriber(Agent):
-    def __init__(self, *, participant, doc: MeshDocument, stt):
-        super().__init__(
-            instructions="not-needed",
-            stt=stt,
-        )
-        self.doc = doc
-        self.participant = participant
-
-    async def on_user_turn_completed(
-        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage
-    ):
-        segments = self.doc.root
-        segments.append_child(
-            "segment",
-            {
-                "text": new_message.text_content,
-                "participant_name": self.participant.name,
-                "participant_id": self.participant.sid,
-                "time": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
-            },
-        )
-        raise StopResponse()
-
-
 class MultiUserTranscriber:
-    def __init__(self, ctx: VoiceConnection, doc: MeshDocument, vad, stt):
+    def __init__(self, ctx: VoiceConnection, path: str, room, vad, stt):
         self.ctx = ctx
-        self.doc = doc
+        self.path = path
+        self.room = room
         self.vad = vad
         self.stt = stt
         self._sessions: dict[str, AgentSession] = {}
@@ -240,6 +214,8 @@ class MultiUserTranscriber:
             *[self._close_session(session) for session in self._sessions.values()]
         )
 
+        self._sessions.clear()
+
         self.ctx.livekit_room.off(
             "participant_connected", self.on_participant_connected
         )
@@ -273,12 +249,38 @@ class MultiUserTranscriber:
         task.add_done_callback(lambda _: self._tasks.discard(task))
 
     async def _start_session(self, participant: rtc.RemoteParticipant) -> AgentSession:
+        logger.info(
+            f"Creating transcription session for {participant.name} ({participant.sid})"
+        )
+
         if participant.identity in self._sessions:
             return self._sessions[participant.identity]
 
+        remote_participant = next(
+            p
+            for p in self.room.messaging.remote_participants
+            if p.id == participant.identity
+        )
+
+        if remote_participant is None:
+            raise RoomException(
+                f"participant was connected to room {participant.identiy}"
+            )
+
         session = AgentSession(
             vad=self.vad,
         )
+
+        agent = Transcriber(
+            events={"user_turn_completed"},
+            transcript_path=self.path,
+            room=self.room,
+            user=remote_participant,
+            agent=self.room.local_participant,
+            instructions="not-needed",
+            stt=self.stt,
+        )
+
         room_io = RoomIO(
             agent_session=session,
             room=self.ctx.livekit_room,
@@ -296,13 +298,7 @@ class MultiUserTranscriber:
             ),
         )
         await room_io.start()
-        await session.start(
-            agent=Transcriber(
-                participant=participant,
-                doc=self.doc,
-                stt=self.stt,
-            )
-        )
+        await session.start(agent=agent)
         return session
 
     async def _close_session(self, sess: AgentSession) -> None:

meshagent/livekit/agents/transcript_logger.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Literal
+from datetime import datetime, timezone
+from livekit.agents import Agent, llm, StopResponse
+from livekit.agents.llm import ChatMessage
+from livekit.agents.voice import ConversationItemAddedEvent
+from meshagent.api import MeshDocument, Participant
+from meshagent.api.room_server_client import RoomClient
+from meshagent.agents.schemas.transcript import transcript_schema
+
+logger = logging.getLogger("transcript_logger")
+
+
+class Transcriber(Agent):
+    def __init__(
+        self,
+        *,
+        user: Participant,
+        agent: Participant,
+        events: set[Literal["user_turn_completed", "conversation_item_added"]],
+        transcript_path: str,
+        room: RoomClient,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.user = user
+        self.agent = agent
+        self.events = events
+        self.transcript_path = transcript_path
+        self.room = room
+        self.doc: MeshDocument | None = None
+
+    def _append_segment(self, *, role: str, text: str, created_at: float):
+        if not text:
+            return
+
+        if not self.doc:
+            logger.warning("Cannot append segment, document not opened")
+            return
+
+        if role == "user":
+            participant_id = self.user.id
+            participant_name = self.user.get_attribute("name")
+        elif role == "assistant":
+            participant_id = self.agent.id
+            participant_name = self.agent.get_attribute("name")
+        else:
+            # skip system / developer / function_call, etc
+            return
+
+        try:
+            segments = self.doc.root
+            segments.append_child(
+                "segment",
+                {
+                    "text": text,
+                    "participant_name": participant_name,
+                    "participant_id": participant_id,
+                    "time": datetime.fromtimestamp(created_at, tz=timezone.utc)
+                    .isoformat()
+                    .replace("+00:00", "Z"),
+                },
+            )
+        except Exception as e:
+            logger.error(f"Failed to write sgement: {e}", exc_info=e)
+
+    def _on_conversation_item(self, event: ConversationItemAddedEvent):
+        item = event.item
+        # Only care about ChatMessages (ignore tool calls, etc.)
+        if not isinstance(item, ChatMessage):
+            return
+
+        text = item.text_content
+        if text is None:
+            return
+
+        # item.role is Literal["developer", "system", "user", "assistant"]
+        self._append_segment(role=item.role, text=text, created_at=item.created_at)
+
+    async def on_enter(self):
+        """Open MeshDocument and attach handler for voicebot transcriptions"""
+        try:
+            self.doc = await self.room.sync.open(
+                path=self.transcript_path, create=True, schema=transcript_schema
+            )
+        except Exception as e:
+            logger.warning(
+                f"Failed to open transcript doc: {e}. This meeting will not be transcribed."
+            )
+            return
+
+        if "conversation_item_added" in self.events:
+            self.session.on("conversation_item_added", self._on_conversation_item)
+
+    async def on_exit(self):
+        """Cleanup handler when voicebot transcription ends and close MeshDocument"""
+        if "conversation_item_added" in self.events:
+            try:
+                self.session.off("conversation_item_added", self._on_conversation_item)
+            except Exception as e:
+                logger.warning(f"Failed to remove handler: {e}")
+        # only close doc if it was successfully opened
+        if self.doc:
+            try:
+                await self.room.sync.close(path=self.transcript_path)
+                logger.info("transcript saved at %s", self.transcript_path)
+            except Exception as e:
+                logger.warning("failed to close transcript doc: %s", e)
+
+    async def on_user_turn_completed(
+        self, turn_ctx: llm.ChatContext, new_message: llm.ChatMessage
+    ):
+        if "user_turn_completed" in self.events:
+            self._append_segment(
+                role="user",
+                text=new_message.text_content,
+                created_at=new_message.created_at,
+            )
+            raise StopResponse()

meshagent/livekit/agents/voice.py

@@ -1,6 +1,7 @@
 import logging
 import asyncio
 from asyncio import CancelledError
+from dataclasses import dataclass
 
 from meshagent.api import (
     RoomMessage,
@@ -20,6 +21,7 @@ from livekit.agents.llm import RawFunctionTool, ToolError, function_tool
 
 from meshagent.openai.proxy import get_client
 from meshagent.agents import AgentChatContext
+from meshagent.livekit.agents.transcript_logger import Transcriber
 from livekit.agents import (
     BackgroundAudioPlayer,
     AudioConfig,
@@ -31,20 +33,11 @@ from livekit.agents import (
 from livekit.plugins import openai, silero
 # from livekit.plugins.turn_detector.multilingual import MultilingualModel
 
-
 import json
 
-from typing import Any
-
-
+from typing import Any, Optional
 from livekit import rtc
-
-from typing import Optional
-
-
 from meshagent.agents import SingleRoomAgent
-
-
 import re
 
 logger = logging.getLogger("voice")
@@ -107,6 +100,14 @@ class VoiceConnection:
         await self.livekit_room.disconnect()
 
 
+@dataclass
+class VoiceBotContext:
+    room: RoomClient
+    caller: Participant
+    connection: VoiceConnection
+    transcript_path: Optional[str] = None
+
+
 class VoiceBot(SingleRoomAgent):
     def __init__(
         self,
@@ -155,6 +156,9 @@ class VoiceBot(SingleRoomAgent):
     def on_message(self, message: RoomMessage):
         if message.type == "voice_call":
             breakout_room = message.message["breakout_room"]
+            transcript_path = message.message.get(
+                "transcript_path"
+            )  # get the transcript_path if the client sends one
 
             logger.info(f"joining breakout room {breakout_room}")
 
@@ -167,10 +171,15 @@ class VoiceBot(SingleRoomAgent):
                     logger.error(f"{e}", exc_info=e)
 
             for participant in self.room.messaging.remote_participants:
-                if participant.id == message.from_participant_id:
+                if (
+                    participant.id == message.message.get("participant_id")
+                    or participant.id == message.from_participant_id
+                ):
                     task = asyncio.create_task(
                         self.run_voice_agent(
-                            participant=participant, breakout_room=breakout_room
+                            participant=participant,
+                            breakout_room=breakout_room,
+                            transcript_path=transcript_path,
                         )
                     )
                     task.add_done_callback(on_done)
@@ -234,7 +243,7 @@ class VoiceBot(SingleRoomAgent):
                     tool_result = await tool.execute(context=context, **raw_arguments)
                 except Exception as e:
                     logger.error(f"failed to call tool {tool.name}: {e}")
-                    return ToolError("f{e}")
+                    return ToolError(f"{e}")
                 if self.tool_adapter is None:
                     if isinstance(tool_result, ErrorResponse):
                         raise ToolError(tool_result.text)
@@ -277,38 +286,51 @@ class VoiceBot(SingleRoomAgent):
             },
         )
 
-    async def create_agent(self, *, context: ToolContext, session: AgentSession):
+    async def create_agent(
+        self,
+        *,
+        context: VoiceBotContext,
+        session: AgentSession,
+    ):
         ctx = ChatContext()
 
         initial_context = await self.init_chat_context()
 
-        rules = await self.get_rules(participant=context.caller)
+        tool_context = ToolContext(
+            room=context.room,
+            caller=context.room.local_participant,
+            on_behalf_of=context.caller,
+        )
+
+        rules = await self.get_rules(participant=context.room.local_participant)
 
         initial_context.replace_rules(rules)
 
         for message in initial_context.messages:
             ctx.add_message(role=message["role"], content=message["content"])
 
-        return Agent(
-            chat_ctx=ctx,
-            instructions="\n".join(self.rules),
-            allow_interruptions=True,
-            tools=[*await self.make_function_tools(context=context)],
-        )
-
-        # agent = Agent(
-        #    instructions="""
-        #        You are a helpful assistant communicating through voice.
-        #    """,
-        #    stt=openai.STT(),
-        #    llm=openai.LLM(model="gpt-4o"),
-        #    tts=openai.TTS(),
-        #    vad=silero.VAD.load(),
-        #   allow_interruptions=True
-        # )
+        if context.transcript_path:
+            return Transcriber(
+                room=context.room,
+                transcript_path=context.transcript_path,
+                events={"conversation_item_added"},
+                user=context.caller,
+                agent=context.room.local_participant,
+                chat_ctx=ctx,
+                instructions=initial_context.get_system_instructions(),
+                allow_interruptions=True,
+                tools=[*await self.make_function_tools(context=tool_context)],
+            )
+        else:
+            return Agent(
+                chat_ctx=ctx,
+                instructions=initial_context.get_system_instructions(),
+                allow_interruptions=True,
+                tools=[*await self.make_function_tools(context=tool_context)],
+            )
 
     async def get_rules(self, *, participant: Participant):
-        rules = [*self._rules]
+        rules = [*self.rules]
         client = participant.get_attribute("client")
 
         if self.client_rules is not None and client is not None:
@@ -321,7 +343,7 @@ class VoiceBot(SingleRoomAgent):
     async def init_chat_context(self) -> AgentChatContext:
         return AgentChatContext()
 
-    def create_session(self, *, context: ToolContext) -> AgentSession:
+    def create_session(self, *, context: VoiceBotContext) -> AgentSession:
         oaiclient = get_client(room=self.room)
 
         session = AgentSession(
@@ -335,55 +357,86 @@ class VoiceBot(SingleRoomAgent):
         )
         return session
 
-    async def run_voice_agent(self, *, participant: Participant, breakout_room: str):
+    async def on_session_created(
+        self, *, context: VoiceBotContext, session: AgentSession
+    ) -> None:
+        return
+
+    async def on_session_started(
+        self, *, context: VoiceBotContext, session: AgentSession
+    ) -> None:
+        return
+
+    async def on_session_ended(
+        self, *, context: VoiceBotContext, session: AgentSession
+    ) -> None:
+        return
+
+    async def run_voice_agent(
+        self,
+        *,
+        participant: Participant,
+        breakout_room: str,
+        transcript_path: Optional[str] = None,
+    ):
         async with VoiceConnection(
             room=self.room, breakout_room=breakout_room
         ) as connection:
             logger.info("starting voice agent")
 
-            context = ToolContext(
+            context = VoiceBotContext(
                 room=self.room,
-                caller=self.room.local_participant,
-                on_behalf_of=participant,
+                caller=participant,
+                connection=connection,
+                transcript_path=transcript_path,
             )
 
             session = self.create_session(context=context)
 
-            agent = await self.create_agent(context=context, session=session)
+            try:
+                await self.on_session_created(context=context, session=session)
 
-            background_audio = BackgroundAudioPlayer(
-                thinking_sound=[
-                    # AudioConfig(
-                    #    os.path.dirname(os.path.abspath(__file__)) +"/sfx/thinking.mp3", volume=0.2),
-                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING, volume=0.3),
-                    AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING2, volume=0.4),
-                ],
-            )
-            await background_audio.start(
-                room=connection.livekit_room, agent_session=session
-            )
+                agent = await self.create_agent(context=context, session=session)
 
-            await session.start(
-                agent=agent,
-                room=connection.livekit_room,
-                room_input_options=RoomInputOptions(
-                    text_enabled=False,
-                    delete_room_on_close=False,
-                ),
-                room_output_options=RoomOutputOptions(
-                    transcription_enabled=True,
-                    audio_enabled=True,
-                ),
-            )
+                background_audio = BackgroundAudioPlayer(
+                    thinking_sound=[
+                        # AudioConfig(
+                        #    os.path.dirname(os.path.abspath(__file__)) +"/sfx/thinking.mp3", volume=0.2),
+                        AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING, volume=0.3),
+                        AudioConfig(BuiltinAudioClip.KEYBOARD_TYPING2, volume=0.4),
+                    ],
+                )
+                await background_audio.start(
+                    room=connection.livekit_room, agent_session=session
+                )
+
+                await session.start(
+                    agent=agent,
+                    room=connection.livekit_room,
+                    room_input_options=RoomInputOptions(
+                        text_enabled=False,
+                        delete_room_on_close=False,
+                    ),
+                    room_output_options=RoomOutputOptions(
+                        transcription_enabled=True,
+                        audio_enabled=True,
+                    ),
+                )
+
+                await self.on_session_started(context=context, session=session)
+                if self.auto_greet_prompt is not None:
+                    session.generate_reply(user_input=self.auto_greet_prompt)
 
-            if self.auto_greet_prompt is not None:
-                session.generate_reply(user_input=self.auto_greet_prompt)
+                if self.auto_greet_message is not None:
+                    session.say(self.auto_greet_message)
 
-            if self.auto_greet_message is not None:
-                session.say(self.auto_greet_message)
+                logger.info("started voice agent")
+                await self._wait_for_disconnect(room=connection.livekit_room)
+
+            except Exception as e:
+                logger.error(f"Error during voice session: {e}", exc_info=e)
 
-            logger.info("started voice agent")
-            await self._wait_for_disconnect(room=connection.livekit_room)
+            await self.on_session_ended(context=context, session=session)
 
 
 class Voicebot(VoiceBot):

meshagent/livekit/version.py

@@ -1 +1 @@
-__version__ = "0.8.3"
+__version__ = "0.20.1"

{meshagent_livekit-0.8.3.dist-info → meshagent_livekit-0.20.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshagent-livekit
-Version: 0.8.3
+Version: 0.20.1
 Summary: Livekit support for Meshagent
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://docs.meshagent.com
@@ -12,13 +12,14 @@ License-File: LICENSE
 Requires-Dist: pytest~=8.4
 Requires-Dist: pytest-asyncio~=0.26
 Requires-Dist: strip-markdown~=1.3
-Requires-Dist: livekit-api>=1.0
-Requires-Dist: livekit-agents~=1.2
-Requires-Dist: livekit-plugins-openai~=1.2
-Requires-Dist: livekit-plugins-silero~=1.2
-Requires-Dist: livekit-plugins-turn-detector~=1.2
-Requires-Dist: meshagent-api~=0.8.3
-Requires-Dist: meshagent-tools~=0.8.3
+Requires-Dist: livekit==1.0.20
+Requires-Dist: livekit-api~=1.1
+Requires-Dist: livekit-agents~=1.3
+Requires-Dist: livekit-plugins-openai~=1.3
+Requires-Dist: livekit-plugins-silero~=1.3
+Requires-Dist: livekit-plugins-turn-detector~=1.3
+Requires-Dist: meshagent-api~=0.20.1
+Requires-Dist: meshagent-tools~=0.20.1
 Dynamic: license-file
 
 # [Meshagent](https://www.meshagent.com)

meshagent_livekit-0.20.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+meshagent/livekit/__init__.py,sha256=X78Z4yEg5XfkNKH0HiIdG4k1q5ktB-ampTuXHLNFrAw,58
+meshagent/livekit/livekit_protocol.py,sha256=5Zu4ymLWEGt5SGXLNu94gOeyjnjhaV6uTS2FhSdODqs,1470
+meshagent/livekit/livekit_protocol_test.py,sha256=o7yYxXad4tMazcxFkq44yW-A9tJ0Lk6WdZpG5ifxcU4,2980
+meshagent/livekit/version.py,sha256=vLeo9ejlCoqrCMhLGa4A6lZVsLnza_kfQ2IJBDs4X90,23
+meshagent/livekit/agents/meeting_transcriber.py,sha256=7dZEuUUpVFHHltrcZ9T4ei5UcKGXvF9uTNRZjbZYm8M,10038
+meshagent/livekit/agents/transcriber.py,sha256=S992oVVBt3ShWDQQWprLjyl6Yh0hyNRd8d3qCmg_toU,5795
+meshagent/livekit/agents/transcript_logger.py,sha256=kRESlbvS7CgrMzTrDhhlaMYEY2bUS2VTxluTurWZ0TM,4299
+meshagent/livekit/agents/voice.py,sha256=f3zm9EfTSCV_0V5VMkEfaZ8ByEJed26picWASDXoDmc,14495
+meshagent_livekit-0.20.1.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_livekit-0.20.1.dist-info/METADATA,sha256=LL1ubfXnlC6CpcHX5n1HY3I0e-6YLXsN6ri7_6utwo0,1783
+meshagent_livekit-0.20.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+meshagent_livekit-0.20.1.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_livekit-0.20.1.dist-info/RECORD,,

meshagent_livekit-0.8.3.dist-info/RECORD

@@ -1,12 +0,0 @@
-meshagent/livekit/__init__.py,sha256=X78Z4yEg5XfkNKH0HiIdG4k1q5ktB-ampTuXHLNFrAw,58
-meshagent/livekit/livekit_protocol.py,sha256=5Zu4ymLWEGt5SGXLNu94gOeyjnjhaV6uTS2FhSdODqs,1470
-meshagent/livekit/livekit_protocol_test.py,sha256=o7yYxXad4tMazcxFkq44yW-A9tJ0Lk6WdZpG5ifxcU4,2980
-meshagent/livekit/version.py,sha256=otnwfmvJLUamPajTPUaIekiO9mA-2HPi-_h_E0N-uOQ,22
-meshagent/livekit/agents/meeting_transcriber.py,sha256=4B5Qi0vKP0TU00vR4KsNdIiR2uyvfgiOL2hjuzhLotw,10210
-meshagent/livekit/agents/transcriber.py,sha256=S992oVVBt3ShWDQQWprLjyl6Yh0hyNRd8d3qCmg_toU,5795
-meshagent/livekit/agents/voice.py,sha256=Fayrgi9JYNDVSGLEnMbBpzkrQF70P5RrfUHSyd1WKwM,12417
-meshagent_livekit-0.8.3.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
-meshagent_livekit-0.8.3.dist-info/METADATA,sha256=QpqzZpHIAZHRj91j7ncNhgKfbanuNBEuWEghgFBq8qY,1749
-meshagent_livekit-0.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-meshagent_livekit-0.8.3.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
-meshagent_livekit-0.8.3.dist-info/RECORD,,