meshagent-livekit 0.5.18__py3-none-any.whl → 0.6.1__py3-none-any.whl

meshagent/livekit/agents/meeting_transcriber.py

@@ -0,0 +1,294 @@
+import asyncio
+import logging
+
+from typing import Optional
+
+from datetime import datetime, timezone
+from livekit import rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    RoomInputOptions,
+    RoomIO,
+    RoomOutputOptions,
+    StopResponse,
+    llm,
+    utils,
+)
+from livekit.plugins import openai, silero
+from meshagent.api import MeshDocument, SchemaRegistration, SchemaRegistry
+from meshagent.agents import SingleRoomAgent
+from meshagent.tools import RemoteToolkit, ToolContext, Tool
+from meshagent.api.room_server_client import Requirement
+from meshagent.livekit.agents.voice import VoiceConnection
+from meshagent.agents.schemas.transcript import transcript_schema
+
+logger = logging.getLogger("meeting_transcriber")
+
+
+class StartTranscriptionTool(Tool):
+    def __init__(self, *, transcriber: "MeetingTranscriber"):
+        self.transcriber = transcriber
+        super().__init__(
+            name="start_transcription",
+            input_schema={
+                "type": "object",
+                "required": [
+                    "breakout_room",
+                    "path",
+                ],
+                "additionalProperties": False,
+                "properties": {
+                    "breakout_room": {
+                        "type": "string",
+                    },
+                    "path": {
+                        "type": "string",
+                    },
+                },
+            },
+        )
+
+    async def execute(self, context: ToolContext, *, breakout_room: str, path: str):
+        await self.transcriber.start_transcription(
+            breakout_room=breakout_room, path=path
+        )
+        return {"status": "started"}
+
+
+class StopTranscriptionTool(Tool):
+    def __init__(self, *, transcriber: "MeetingTranscriber"):
+        self.transcriber = transcriber
+        super().__init__(
+            name="stop_transcription",
+            input_schema={
+                "type": "object",
+                "required": [
+                    "breakout_room",
+                ],
+                "additionalProperties": False,
+                "properties": {
+                    "breakout_room": {
+                        "type": "string",
+                    },
+                },
+            },
+        )
+
+    async def execute(self, context: ToolContext, *, breakout_room: str):
+        await self.transcriber.stop_transcription(
+            breakout_room=breakout_room,
+        )
+        return {"status": "stopped"}
+
+
+class MeetingTranscriber(SingleRoomAgent):
+    def __init__(self, name: str, requires: Optional[list[Requirement]] = None):
+        super().__init__(
+            name=name,
+            requires=requires,
+        )
+        self._toolkit = RemoteToolkit(
+            name="transcription",
+            tools=[
+                StartTranscriptionTool(transcriber=self),
+                StopTranscriptionTool(transcriber=self),
+            ],
+        )
+        self._vad = None
+        self._transcription_tasks = dict[str, tuple[asyncio.Task, asyncio.Future]]()
+
+    async def start(self, *, room):
+        await super().start(room=room)
+        await self._toolkit.start(room=room)
+        await room.local_participant.set_attribute("supports_voice", True)
+        await room.messaging.enable()
+
+        self._vad = silero.VAD.load()
+
+    async def start_transcription(self, *, breakout_room: Optional[str], path: str):
+        stop_fut = asyncio.Future()
+
+        async def transcribe():
+            await self.room.local_participant.set_attribute(
+                f"transcribing.{breakout_room}", True
+            )
+
+            try:
+                async with VoiceConnection(
+                    room=self.room, breakout_room=breakout_room
+                ) as conn:
+                    doc = await self.room.sync.open(path=path, create=True)
+
+                    transcriber = MultiUserTranscriber(conn, doc, self._vad)
+                    transcriber.start()
+
+                    for participant in conn.livekit_room.remote_participants.values():
+                        # handle all existing participants
+                        transcriber.on_participant_connected(participant)
+
+                    await stop_fut
+
+                    await self.room.local_participant.set_attribute(
+                        f"transcribing.{breakout_room}", False
+                    )
+
+                    await self.room.sync.close(path=path)
+
+                    await transcriber.aclose()
+            except Exception as ex:
+                logger.error(f"error during transcription {ex}", exc_info=ex)
+                pass
+
+            await self.room.local_participant.set_attribute(
+                f"transcribing.{breakout_room}", False
+            )
+
+        if breakout_room not in self._transcription_tasks:
+            self._transcription_tasks[breakout_room] = (
+                asyncio.create_task(transcribe()),
+                stop_fut,
+            )
+
+    async def stop_transcription(self, *, breakout_room: Optional[str]):
+        if breakout_room in self._transcription_tasks:
+            task, fut = self._transcription_tasks.pop(breakout_room)
+            fut.set_result(True)
+            await asyncio.gather(task)
+
+    async def stop(self):
+        await self._toolkit.stop()
+
+        tasks = []
+        for breakout_room, _ in self._transcription_tasks.items():
+            task, fut = self._transcription_tasks.pop(breakout_room)
+            fut.set_result(True)
+            tasks.append(task)
+
+        await asyncio.gather(*tasks)
+        await super().stop()
+
+
+class TranscriptRegistry(SchemaRegistry):
+    def __init__(self):
+        name = "transcript"
+        super().__init__(
+            name=f"meshagent.schema.{name}",
+            validate_webhook_secret=False,
+            schemas=[SchemaRegistration(name=name, schema=transcript_schema)],
+        )
+
+
+class Transcriber(Agent):
+    def __init__(self, *, participant, doc: MeshDocument):
+        super().__init__(
+            instructions="not-needed",
+            stt=openai.STT(),
+        )
+        self.doc = doc
+        self.participant = participant
+
+    async def on_user_turn_completed(
+        self, chat_ctx: llm.ChatContext, new_message: llm.ChatMessage
+    ):
+        segments = self.doc.root
+        segments.append_child(
+            "segment",
+            {
+                "text": new_message.text_content,
+                "participant_name": self.participant.name,
+                "participant_id": self.participant.sid,
+                "time": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+            },
+        )
+        raise StopResponse()
+
+
+class MultiUserTranscriber:
+    def __init__(self, ctx: VoiceConnection, doc: MeshDocument, vad):
+        self.ctx = ctx
+        self.doc = doc
+        self.vad = vad
+        self._sessions: dict[str, AgentSession] = {}
+        self._tasks: set[asyncio.Task] = set()
+
+    def start(self):
+        self.ctx.livekit_room.on("participant_connected", self.on_participant_connected)
+        self.ctx.livekit_room.on(
+            "participant_disconnected", self.on_participant_disconnected
+        )
+
+    async def aclose(self):
+        await utils.aio.cancel_and_wait(*self._tasks)
+
+        await asyncio.gather(
+            *[self._close_session(session) for session in self._sessions.values()]
+        )
+
+        self.ctx.livekit_room.off(
+            "participant_connected", self.on_participant_connected
+        )
+        self.ctx.livekit_room.off(
+            "participant_disconnected", self.on_participant_disconnected
+        )
+
+    def on_participant_connected(self, participant: rtc.RemoteParticipant):
+        if participant.identity in self._sessions:
+            return
+
+        logger.info(f"starting session for {participant.identity}")
+        task = asyncio.create_task(self._start_session(participant))
+        self._tasks.add(task)
+
+        def on_task_done(task: asyncio.Task):
+            try:
+                self._sessions[participant.identity] = task.result()
+            finally:
+                self._tasks.discard(task)
+
+        task.add_done_callback(on_task_done)
+
+    def on_participant_disconnected(self, participant: rtc.RemoteParticipant):
+        if (session := self._sessions.pop(participant.identity)) is None:
+            return
+
+        logger.info(f"closing session for {participant.identity}")
+        task = asyncio.create_task(self._close_session(session))
+        self._tasks.add(task)
+        task.add_done_callback(lambda _: self._tasks.discard(task))
+
+    async def _start_session(self, participant: rtc.RemoteParticipant) -> AgentSession:
+        if participant.identity in self._sessions:
+            return self._sessions[participant.identity]
+
+        session = AgentSession(
+            vad=self.vad,
+        )
+        room_io = RoomIO(
+            agent_session=session,
+            room=self.ctx.livekit_room,
+            participant=participant,
+            input_options=RoomInputOptions(
+                # text input is not supported for multiple room participants
+                # if needed, register the text stream handler by yourself
+                # and route the text to different sessions based on the participant identity
+                text_enabled=False,
+                delete_room_on_close=False,
+            ),
+            output_options=RoomOutputOptions(
+                transcription_enabled=True,
+                audio_enabled=False,
+            ),
+        )
+        await room_io.start()
+        await session.start(
+            agent=Transcriber(
+                participant=participant,
+                doc=self.doc,
+            )
+        )
+        return session
+
+    async def _close_session(self, sess: AgentSession) -> None:
+        await sess.drain()
+        await sess.aclose()

meshagent/livekit/version.py

@@ -1 +1 @@
-__version__ = "0.5.18"
+__version__ = "0.6.1"

{meshagent_livekit-0.5.18.dist-info → meshagent_livekit-0.6.1.dist-info}/METADATA

@@ -1,24 +1,24 @@
 Metadata-Version: 2.4
 Name: meshagent-livekit
-Version: 0.5.18
+Version: 0.6.1
 Summary: Livekit support for Meshagent
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://docs.meshagent.com
 Project-URL: Website, https://www.meshagent.com
 Project-URL: Source, https://www.meshagent.com
-Requires-Python: >=3.12
+Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pytest~=8.4
 Requires-Dist: pytest-asyncio~=0.26
 Requires-Dist: strip-markdown~=1.3
 Requires-Dist: livekit-api>=1.0
-Requires-Dist: livekit-agents~=1.1
-Requires-Dist: livekit-plugins-openai~=1.1
-Requires-Dist: livekit-plugins-silero~=1.1
-Requires-Dist: livekit-plugins-turn-detector~=1.1
-Requires-Dist: meshagent-api~=0.5.18
-Requires-Dist: meshagent-tools~=0.5.18
+Requires-Dist: livekit-agents~=1.2
+Requires-Dist: livekit-plugins-openai~=1.2
+Requires-Dist: livekit-plugins-silero~=1.2
+Requires-Dist: livekit-plugins-turn-detector~=1.2
+Requires-Dist: meshagent-api~=0.6.1
+Requires-Dist: meshagent-tools~=0.6.1
 Dynamic: license-file
 
 # [Meshagent](https://www.meshagent.com)
@@ -27,7 +27,7 @@ Dynamic: license-file
 The ``meshagent.livekit`` package equips agents with real-time audio and voice capabilities via the LiveKit SDK. 
 
 ### VoiceBot
-The ``VoiceBot`` agent handles two-way voice conversations allowing users to interact with the agent verbally. Agents based on the ``VoiceBot`` class can be given the same tools as ``ChatBot`` based agents. This means you only need to write a tool once and the same tool can be used across both text and voice based agents. Check out the [Build and Deploy a Voice Agent](https://docs.meshagent.com/agents/standard/buildanddeployvoicebot) example to learn how to create a simple Voice Agent without tools then add built-in MeshAgent tools and custom tools to the agent.
+The ``VoiceBot`` agent handles two-way voice conversations allowing users to interact with the agent verbally. Agents based on the ``VoiceBot`` class can be given the same tools as ``ChatBot`` based agents. This means you only need to write a tool once and the same tool can be used across both text and voice based agents. Check out the [Build a Voice Agent](https://docs.meshagent.com/agents/standard/buildanddeployvoicebot) example to learn how to create a simple Voice Agent without tools then add built-in MeshAgent tools and custom tools to the agent.
 
 ---
 ### Learn more about MeshAgent on our website or check out the docs for additional examples!

meshagent_livekit-0.6.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+meshagent/livekit/__init__.py,sha256=X78Z4yEg5XfkNKH0HiIdG4k1q5ktB-ampTuXHLNFrAw,58
+meshagent/livekit/livekit_protocol.py,sha256=5Zu4ymLWEGt5SGXLNu94gOeyjnjhaV6uTS2FhSdODqs,1470
+meshagent/livekit/livekit_protocol_test.py,sha256=o7yYxXad4tMazcxFkq44yW-A9tJ0Lk6WdZpG5ifxcU4,2980
+meshagent/livekit/version.py,sha256=baAcEjLSYFIeNZF51tOMmA_zAMhN8HvKael-UU-Ruec,22
+meshagent/livekit/agents/meeting_transcriber.py,sha256=_DHFCuOGCE3LBBBDoF1xGTBMHXaVm2yzqya-OO4g6xg,9778
+meshagent/livekit/agents/transcriber.py,sha256=S992oVVBt3ShWDQQWprLjyl6Yh0hyNRd8d3qCmg_toU,5795
+meshagent/livekit/agents/voice.py,sha256=STgjMSqzUgV9UAmleOy1vkgRXP93MDSYgiOO6Lo0peU,11964
+meshagent_livekit-0.6.1.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_livekit-0.6.1.dist-info/METADATA,sha256=_z2iTjyR9cJEo2Y_6IMJGSD5kKs6-cm7Cf23Qb7b2wo,1749
+meshagent_livekit-0.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+meshagent_livekit-0.6.1.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_livekit-0.6.1.dist-info/RECORD,,

meshagent_livekit-0.5.18.dist-info/RECORD

@@ -1,11 +0,0 @@
-meshagent/livekit/__init__.py,sha256=X78Z4yEg5XfkNKH0HiIdG4k1q5ktB-ampTuXHLNFrAw,58
-meshagent/livekit/livekit_protocol.py,sha256=5Zu4ymLWEGt5SGXLNu94gOeyjnjhaV6uTS2FhSdODqs,1470
-meshagent/livekit/livekit_protocol_test.py,sha256=o7yYxXad4tMazcxFkq44yW-A9tJ0Lk6WdZpG5ifxcU4,2980
-meshagent/livekit/version.py,sha256=8HarzfUUN0oBs56RqaWFFfXKWTGi5Mt2jU6YSekUkis,23
-meshagent/livekit/agents/transcriber.py,sha256=S992oVVBt3ShWDQQWprLjyl6Yh0hyNRd8d3qCmg_toU,5795
-meshagent/livekit/agents/voice.py,sha256=STgjMSqzUgV9UAmleOy1vkgRXP93MDSYgiOO6Lo0peU,11964
-meshagent_livekit-0.5.18.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
-meshagent_livekit-0.5.18.dist-info/METADATA,sha256=pOGhINg_FthW2_8GHEUdunQm3Yn6kW-pTmBQIiowwmY,1763
-meshagent_livekit-0.5.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-meshagent_livekit-0.5.18.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
-meshagent_livekit-0.5.18.dist-info/RECORD,,