meshagent-agents 0.5.15__py3-none-any.whl → 0.6.0__py3-none-any.whl

meshagent/agents/chat.py

@@ -4,21 +4,28 @@ from meshagent.api import (
     RoomMessage,
     RoomClient,
     RemoteParticipant,
+    Participant,
     RequiredSchema,
     Requirement,
     Element,
     MeshDocument,
 )
-from meshagent.tools import Toolkit, ToolContext
+from meshagent.tools import Toolkit, ToolContext, make_tools, ToolkitBuilder
 from meshagent.agents.adapter import LLMAdapter, ToolResponseAdapter
-from meshagent.openai.tools.responses_adapter import ImageGenerationTool, LocalShellTool
+from meshagent.openai.tools.responses_adapter import (
+    ImageGenerationConfig,
+    ImageGenerationTool,
+    LocalShellConfig,
+    LocalShellTool,
+    #    WebSearchConfig,
+    #    WebSearchTool,
+    ReasoningTool,
+)
 import asyncio
 from typing import Optional
 import logging
-from meshagent.tools import MultiToolkit
 import uuid
 import datetime
-from typing import Literal
 import base64
 from openai.types.responses import ResponseStreamEvent
 from asyncio import CancelledError
@@ -32,10 +39,108 @@ tracer = trace.get_tracer("meshagent.chatbot")
 logger = logging.getLogger("chat")
 
 
-class ChatBotThreadLocalShellTool(LocalShellTool):
-    def __init__(self, *, thread_context: "ChatThreadContext"):
+class ChatBotReasoningTool(ReasoningTool):
+    def __init__(self, *, room: RoomClient, thread_context: "ChatThreadContext"):
         super().__init__()
         self.thread_context = thread_context
+        self.room = room
+
+        self._reasoning_element = None
+        self._reasoning_item = None
+
+    def _get_messages_element(self):
+        messages = self.thread_context.thread.root.get_children_by_tag_name("messages")
+        if len(messages) > 0:
+            return messages[0]
+        return None
+
+    async def on_reasoning_summary_part_added(
+        self,
+        context: ToolContext,
+        *,
+        item_id: str,
+        output_index: int,
+        part: dict,
+        sequence_number: int,
+        summary_index: int,
+        type: str,
+        **extra,
+    ):
+        el = self._get_messages_element()
+        if el is None:
+            logger.warning("missing messages element, cannot log reasoning")
+        else:
+            self._reasoning_element = el.append_child("reasoning", {"summary": ""})
+
+    async def on_reasoning_summary_part_done(
+        self,
+        context: ToolContext,
+        *,
+        item_id: str,
+        output_index: int,
+        part: dict,
+        sequence_number: int,
+        summary_index: int,
+        type: str,
+        **extra,
+    ):
+        self._reasoning_element = None
+
+    async def on_reasoning_summary_text_delta(
+        self,
+        context: ToolContext,
+        *,
+        delta: str,
+        output_index: int,
+        sequence_number: int,
+        summary_index: int,
+        type: str,
+        **extra,
+    ):
+        el = self._reasoning_element
+        el.set_attribute("summary", el.get_attribute("summary") + delta)
+
+    async def on_reasoning_summary_text_done(
+        self,
+        context: ToolContext,
+        *,
+        item_id: str,
+        output_index: int,
+        sequence_number: int,
+        summary_index: int,
+        type: str,
+        **extra,
+    ):
+        pass
+
+
+class ChatBotThreadLocalShellToolkitBuilder(ToolkitBuilder):
+    def __init__(self, *, thread_context: "ChatThreadContext"):
+        super().__init__(name="local_shell", type=ImageGenerationConfig)
+        self.thread_context = thread_context
+
+    def make(
+        self,
+        *,
+        model: str,
+        config: LocalShellConfig,
+    ):
+        return Toolkit(
+            name="local_shell",
+            tools=[
+                ChatBotThreadLocalShellTool(
+                    config=config, thread_context=self.thread_context
+                )
+            ],
+        )
+
+
+class ChatBotThreadLocalShellTool(LocalShellTool):
+    def __init__(
+        self, *, thread_context: "ChatThreadContext", config: LocalShellConfig
+    ):
+        super().__init__(config=config)
+        self.thread_context = thread_context
 
     async def execute_shell_command(
         self,
@@ -75,32 +180,35 @@ class ChatBotThreadLocalShellTool(LocalShellTool):
         return result
 
 
+class ChatBotThreadOpenAIImageGenerationToolkitBuilder(ToolkitBuilder):
+    def __init__(self, *, thread_context: "ChatThreadContext"):
+        super().__init__(name="image_generation", type=ImageGenerationConfig)
+        self.thread_context = thread_context
+
+    def make(
+        self,
+        *,
+        model: str,
+        config: ImageGenerationConfig,
+    ):
+        return Toolkit(
+            name="image_generation",
+            tools=[
+                ChatBotThreadOpenAIImageGenerationTool(
+                    config=config, thread_context=self.thread_context
+                )
+            ],
+        )
+
+
 class ChatBotThreadOpenAIImageGenerationTool(ImageGenerationTool):
     def __init__(
         self,
         *,
-        background: Literal["transparent", "opaque", "auto"] = None,
-        input_image_mask_url: Optional[str] = None,
-        model: Optional[str] = None,
-        moderation: Optional[str] = None,
-        output_compression: Optional[int] = None,
-        output_format: Optional[Literal["png", "webp", "jpeg"]] = None,
-        partial_images: Optional[int] = None,
-        quality: Optional[Literal["auto", "low", "medium", "high"]] = None,
-        size: Optional[Literal["1024x1024", "1024x1536", "1536x1024", "auto"]] = None,
+        config: ImageGenerationConfig,
         thread_context: "ChatThreadContext",
     ):
-        super().__init__(
-            background=background,
-            input_image_mask_url=input_image_mask_url,
-            model=model,
-            moderation=moderation,
-            output_compression=output_compression,
-            output_format=output_format,
-            partial_images=partial_images,
-            quality=quality,
-            size=size,
-        )
+        super().__init__(config=config)
 
         self.thread_context = thread_context
 
@@ -214,8 +322,11 @@ class ChatBotThreadOpenAIImageGenerationTool(ImageGenerationTool):
         )
 
 
-def get_thread_participants(
-    *, room: RoomClient, thread: MeshDocument
+def get_online_participants(
+    *,
+    room: RoomClient,
+    thread: MeshDocument,
+    exclude: Optional[list[Participant]] = None,
 ) -> list[RemoteParticipant]:
     results = list[RemoteParticipant]()
 
@@ -224,7 +335,8 @@ def get_thread_participants(
             for member in prop.get_children():
                 for online in room.messaging.get_participants():
                     if online.get_attribute("name") == member.get_attribute("name"):
-                        results.append(online)
+                        if exclude is None or online not in exclude:
+                            results.append(online)
 
     return results
 
@@ -247,7 +359,6 @@ class ChatThreadContext:
         self.path = path
 
 
-# todo: thread should stop when participant stops?
 class ChatBot(SingleRoomAgent):
     def __init__(
         self,
@@ -262,7 +373,8 @@ class ChatBot(SingleRoomAgent):
         rules: Optional[list[str]] = None,
         auto_greet_message: Optional[str] = None,
         empty_state_title: Optional[str] = None,
-        labels: Optional[str] = None,
+        labels: Optional[list[str]] = None,
+        decision_model: Optional[str] = None,
     ):
         super().__init__(
             name=name,
@@ -275,6 +387,10 @@ class ChatBot(SingleRoomAgent):
         if toolkits is None:
             toolkits = []
 
+        self._decision_model = (
+            "gpt-4.1-mini" if decision_model is None else decision_model
+        )
+
         self._llm_adapter = llm_adapter
         self._tool_adapter = tool_adapter
 
@@ -295,6 +411,7 @@ class ChatBot(SingleRoomAgent):
         self._empty_state_title = empty_state_title
 
         self._thread_tasks = dict[str, asyncio.Task]()
+        self._open_threads = {}
 
     def init_requirements(self, requires: list[Requirement]):
         if requires is None:
@@ -373,8 +490,15 @@ class ChatBot(SingleRoomAgent):
                 thread_attributes=thread_attributes,
             )
 
-    async def get_thread_participants(self, *, thread: MeshDocument):
-        return get_thread_participants(room=self._room, thread=thread)
+    async def get_online_participants(
+        self, *, thread: MeshDocument, exclude: Optional[list[Participant]] = None
+    ):
+        return get_online_participants(room=self._room, thread=thread, exclude=exclude)
+
+    async def get_thread_toolkit_builders(
+        self, *, thread_context: ChatThreadContext, participant: RemoteParticipant
+    ) -> list[ToolkitBuilder]:
+        return []
 
     async def get_thread_toolkits(
         self, *, thread_context: ChatThreadContext, participant: RemoteParticipant
@@ -382,27 +506,23 @@ class ChatBot(SingleRoomAgent):
         toolkits = await self.get_required_toolkits(
             context=ToolContext(
                 room=self.room,
-                caller=participant,
+                caller=self.room.local_participant,
+                on_behalf_of=participant,
                 caller_context={"chat": thread_context.chat.to_json()},
             )
         )
-        toaster = None
-
-        for toolkit in toolkits:
-            if toolkit.name == "ui":
-                for tool in toolkit.tools:
-                    if tool.name == "show_toast":
-                        toaster = tool
-
-        if toaster is not None:
 
-            def multi_tool(toolkit: Toolkit):
-                if toaster in toolkit.tools:
-                    return toolkit
-
-                return MultiToolkit(required=[toaster], base_toolkit=toolkit)
-
-            toolkits = list(map(multi_tool, toolkits))
+        toolkits.append(
+            Toolkit(
+                name="reasoning",
+                tools=[
+                    ChatBotReasoningTool(
+                        room=self._room,
+                        thread_context=thread_context,
+                    )
+                ],
+            )
+        )
 
         return [*self._toolkits, *toolkits]
 
@@ -412,9 +532,19 @@ class ChatBot(SingleRoomAgent):
         return context
 
     async def open_thread(self, *, path: str):
-        return await self.room.sync.open(path=path)
+        logger.info(f"opening thread {path}")
+        if path not in self._open_threads:
+            fut = asyncio.ensure_future(self.room.sync.open(path=path))
+            self._open_threads[path] = fut
+
+        return await self._open_threads[path]
 
     async def close_thread(self, *, path: str):
+        logger.info(f"closing thread {path}")
+
+        if path in self._open_threads:
+            del self._open_threads[path]
+
         return await self.room.sync.close(path=path)
 
     async def load_thread_context(self, *, thread_context: ChatThreadContext):
@@ -435,7 +565,9 @@ class ChatBot(SingleRoomAgent):
                         ] == self.room.local_participant.get_attribute("name"):
                             chat_context.append_assistant_message(msg)
                         else:
-                            chat_context.append_user_message(msg)
+                            chat_context.append_user_message(
+                                element["author_name"] + " said: " + msg
+                            )
 
                         for child in element.get_children():
                             if child.tag_name == "file":
@@ -448,7 +580,7 @@ class ChatBot(SingleRoomAgent):
         if doc_messages is None:
             raise Exception("thread was not properly initialized")
 
-    async def prepare_llm_context(self, *, context: ChatThreadContext):
+    async def prepare_llm_context(self, *, thread_context: ChatThreadContext):
         """
         called prior to sending the request to the LLM in case the agent needs to modify the context prior to sending
         """
@@ -552,7 +684,156 @@ class ChatBot(SingleRoomAgent):
             pass
         finally:
             updates.shutdown()
-            await update_thread_task
+
+        await update_thread_task
+
+    def get_thread_members(self, *, thread: MeshDocument) -> list[str]:
+        results = []
+
+        for prop in thread.root.get_children():
+            if prop.tag_name == "members":
+                for member in prop.get_children():
+                    results.append(member.get_attribute("name"))
+
+        return results
+
+    async def should_reply(
+        self,
+        *,
+        context: ChatThreadContext,
+        has_more_than_one_other_user: bool,
+        toolkits: list[Toolkit],
+        from_user: RemoteParticipant,
+        online: list[Participant],
+    ):
+        if not has_more_than_one_other_user:
+            return True
+
+        online_set = {}
+
+        all_members = []
+        online_members = []
+
+        for m in self.get_thread_members(thread=context.thread):
+            all_members.append(m)
+
+        for o in online:
+            if o.get_attribute("name") not in online_set:
+                online_set[o.get_attribute("name")] = True
+                online_members.append(o.get_attribute("name"))
+
+        logger.info(
+            "multiple participants detected, checking whether agent should reply to conversation"
+        )
+
+        cloned_context = context.chat.copy()
+        cloned_context.replace_rules(
+            rules=[
+                "examine the conversation so far and return whether the user is expecting a reply from you or another user as the next message in the conversation",
+                f'your name (the assistant) is "{self.room.local_participant.get_attribute("name")}"',
+                "if the user mentions a person with another name, they aren't talking to you unless they also mention you",
+                "if the user poses a question to everyone, they are talking to you",
+                f"members of thread are currently {all_members}",
+                f"users online currently are {online_members}",
+            ]
+        )
+        response = await self._llm_adapter.next(
+            context=cloned_context,
+            room=self._room,
+            model=self._decision_model or self._llm_adapter.default_model(),
+            on_behalf_of=from_user,
+            toolkits=[],
+            output_schema={
+                "type": "object",
+                "required": ["reasoning", "expecting_assistant_reply", "next_user"],
+                "additionalProperties": False,
+                "properties": {
+                    "reasoning": {
+                        "type": "string",
+                        "description": "explain why you think the user was or was not expecting you to reply",
+                    },
+                    "next_user": {
+                        "type": "string",
+                        "description": "who would be expectd to send the next message in the conversation",
+                    },
+                    "expecting_assistant_reply": {"type": "boolean"},
+                },
+            },
+        )
+
+        logger.info(f"should reply check returned {response}")
+
+        return response["expecting_assistant_reply"]
+
+    async def handle_user_message(
+        self,
+        *,
+        context: ChatThreadContext,
+        toolkits: list[Toolkit],
+        model: str,
+        from_user: RemoteParticipant,
+        event_handler,
+    ):
+        online = await self.get_online_participants(
+            thread=context.thread, exclude=[self.room.local_participant]
+        )
+
+        for participant in get_online_participants(
+            room=self._room, thread=context.thread
+        ):
+            self._room.messaging.send_message_nowait(
+                to=participant,
+                type="listening",
+                message={"listening": True, "path": context.path},
+            )
+
+        has_more_than_one_other_user = False
+
+        for member_name in self.get_thread_members(thread=context.thread):
+            if member_name != self._room.local_participant.get_attribute(
+                "name"
+            ) and member_name != from_user.get_attribute("name"):
+                has_more_than_one_other_user = True
+                break
+
+        reply = await self.should_reply(
+            has_more_than_one_other_user=has_more_than_one_other_user,
+            online=online,
+            context=context,
+            toolkits=toolkits,
+            from_user=from_user,
+        )
+
+        for participant in get_online_participants(
+            room=self._room, thread=context.thread
+        ):
+            self._room.messaging.send_message_nowait(
+                to=participant,
+                type="listening",
+                message={"listening": False, "path": context.path},
+            )
+
+        if not reply:
+            return
+
+        for participant in get_online_participants(
+            room=self._room, thread=context.thread
+        ):
+            self._room.messaging.send_message_nowait(
+                to=participant,
+                type="thinking",
+                message={"thinking": True, "path": context.path},
+            )
+
+        await self._llm_adapter.next(
+            context=context.chat,
+            room=self._room,
+            toolkits=toolkits,
+            tool_adapter=self._tool_adapter,
+            event_handler=event_handler,
+            model=model,
+            on_behalf_of=from_user,
+        )
 
     async def _spawn_thread(self, path: str, messages: Chan[RoomMessage]):
         logger.debug("chatbot is starting a thread", extra={"path": path})
@@ -570,120 +851,108 @@ class ChatBot(SingleRoomAgent):
             received = None
 
             while True:
-                while True:
-                    logger.debug(f"waiting for message on thread {path}")
-                    received = await messages.recv()
-                    logger.debug(f"received message on thread {path}: {received.type}")
-
-                    chat_with_participant = None
-                    for participant in self._room.messaging.get_participants():
-                        if participant.id == received.from_participant_id:
-                            chat_with_participant = participant
-                            break
-
-                    if chat_with_participant is None:
-                        logger.warning(
-                            "participant does not have messaging enabled, skipping message"
-                        )
-                        continue
-
-                    thread_attributes = {
-                        "agent_name": self.name,
-                        "agent_participant_id": self.room.local_participant.id,
-                        "agent_participant_name": self.room.local_participant.get_attribute(
-                            "name"
-                        ),
-                        "remote_participant_id": chat_with_participant.id,
-                        "remote_participant_name": chat_with_participant.get_attribute(
-                            "name"
-                        ),
-                        "path": path,
-                    }
-
-                    if current_file != chat_with_participant.get_attribute(
-                        "current_file"
-                    ):
-                        logger.info(
-                            f"participant is now looking at {chat_with_participant.get_attribute('current_file')}"
-                        )
-                        current_file = chat_with_participant.get_attribute(
-                            "current_file"
-                        )
+                logger.debug(f"waiting for message on thread {path}")
+                received = await messages.recv()
+                logger.debug(f"received message on thread {path}: {received.type}")
 
-                    if current_file is not None:
-                        chat_context.append_assistant_message(
-                            message=f"the user is currently viewing the file at the path: {current_file}"
-                        )
+                chat_with_participant = None
+                for participant in self._room.messaging.get_participants():
+                    if participant.id == received.from_participant_id:
+                        chat_with_participant = participant
+                        break
 
-                    elif current_file is not None:
-                        chat_context.append_assistant_message(
-                            message="the user is not current viewing any files"
-                        )
+                if chat_with_participant is None:
+                    logger.warning(
+                        "participant does not have messaging enabled, skipping message"
+                    )
+                    continue
+
+                thread_attributes = {
+                    "agent_name": self.name,
+                    "agent_participant_id": self.room.local_participant.id,
+                    "agent_participant_name": self.room.local_participant.get_attribute(
+                        "name"
+                    ),
+                    "remote_participant_id": chat_with_participant.id,
+                    "remote_participant_name": chat_with_participant.get_attribute(
+                        "name"
+                    ),
+                    "path": path,
+                }
+
+                if current_file != chat_with_participant.get_attribute("current_file"):
+                    logger.info(
+                        f"participant is now looking at {chat_with_participant.get_attribute('current_file')}"
+                    )
+                    current_file = chat_with_participant.get_attribute("current_file")
 
-                    if thread is None:
-                        with tracer.start_as_current_span(
-                            "chatbot.thread.open"
-                        ) as span:
-                            span.set_attributes(thread_attributes)
-
-                            thread = await self.open_thread(path=path)
-
-                            thread_context = ChatThreadContext(
-                                path=path,
-                                chat=chat_context,
-                                thread=thread,
-                                participants=get_thread_participants(
-                                    room=self.room, thread=thread
-                                ),
-                            )
+                if current_file is not None:
+                    chat_context.append_assistant_message(
+                        message=f"the user is currently viewing the file at the path: {current_file}"
+                    )
 
-                            await self.load_thread_context(
-                                thread_context=thread_context
-                            )
+                elif current_file is not None:
+                    chat_context.append_assistant_message(
+                        message="the user is not current viewing any files"
+                    )
 
-                    if received.type == "opened":
-                        if not opened:
-                            opened = True
+                if thread is None:
+                    with tracer.start_as_current_span("chatbot.thread.open") as span:
+                        span.set_attributes(thread_attributes)
 
-                            await self._greet(
-                                path=path,
-                                chat_context=chat_context,
-                                participant=chat_with_participant,
-                                thread=thread,
-                                thread_attributes=thread_attributes,
-                            )
+                        thread = await self.open_thread(path=path)
 
-                    if received.type == "chat":
-                        if thread is None:
-                            logger.info("thread is not open", extra={"path": path})
-                            break
-
-                        logger.debug(
-                            "chatbot received a chat",
-                            extra={
-                                "context": chat_context.id,
-                                "participant_id": self.room.local_participant.id,
-                                "participant_name": self.room.local_participant.get_attribute(
-                                    "name"
-                                ),
-                                "text": received.message["text"],
-                            },
+                        thread_context = ChatThreadContext(
+                            path=path,
+                            chat=chat_context,
+                            thread=thread,
+                            participants=get_online_participants(
+                                room=self.room, thread=thread
+                            ),
                         )
 
-                        attachments = received.message.get("attachments", [])
-                        text = received.message["text"]
+                        await self.load_thread_context(thread_context=thread_context)
 
-                        for attachment in attachments:
-                            chat_context.append_assistant_message(
-                                message=f"the user attached a file at the path '{attachment['path']}'"
-                            )
+                if received.type == "opened":
+                    if not opened:
+                        opened = True
+
+                        await self._greet(
+                            path=path,
+                            chat_context=chat_context,
+                            participant=chat_with_participant,
+                            thread=thread,
+                            thread_attributes=thread_attributes,
+                        )
+
+                if received.type == "chat":
+                    if thread is None:
+                        logger.info("thread is not open", extra={"path": path})
+                        break
+
+                    logger.debug(
+                        "chatbot received a chat",
+                        extra={
+                            "context": chat_context.id,
+                            "participant_id": self.room.local_participant.id,
+                            "participant_name": self.room.local_participant.get_attribute(
+                                "name"
+                            ),
+                            "text": received.message["text"],
+                        },
+                    )
 
-                        chat_context.append_user_message(message=text)
+                    attachments = received.message.get("attachments", [])
+                    text = received.message["text"]
+
+                    for attachment in attachments:
+                        chat_context.append_assistant_message(
+                            message=f"the user attached a file at the path '{attachment['path']}'"
+                        )
 
-                        if messages.empty():
-                            break
+                    chat_context.append_user_message(message=text)
 
-                if received is not None:
+                if received is not None and received.type == "chat":
                     with tracer.start_as_current_span("chatbot.thread.message") as span:
                         span.set_attributes(thread_attributes)
                         span.set_attribute("role", "user")
@@ -699,27 +968,17 @@ class ChatBot(SingleRoomAgent):
                         span.set_attributes({"text": text})
 
                         try:
-                            for participant in get_thread_participants(
-                                room=self._room, thread=thread
-                            ):
-                                # TODO: async gather
-                                self._room.messaging.send_message_nowait(
-                                    to=participant,
-                                    type="thinking",
-                                    message={"thinking": True, "path": path},
-                                )
-
                             if thread_context is None:
                                 thread_context = ChatThreadContext(
                                     path=path,
                                     chat=chat_context,
                                     thread=thread,
-                                    participants=get_thread_participants(
+                                    participants=get_online_participants(
                                         room=self.room, thread=thread
                                     ),
                                 )
                             else:
-                                thread_context.participants = get_thread_participants(
+                                thread_context.participants = get_online_participants(
                                     room=self.room, thread=thread
                                 )
 
@@ -731,12 +990,22 @@ class ChatBot(SingleRoomAgent):
                                         thread_toolkits = (
                                             await self.get_thread_toolkits(
                                                 thread_context=thread_context,
-                                                participant=participant,
+                                                participant=chat_with_participant,
+                                            )
+                                        )
+
+                                    with tracer.start_as_current_span(
+                                        "get_thread_toolkit_builders"
+                                    ) as span:
+                                        thread_tool_providers = (
+                                            await self.get_thread_toolkit_builders(
+                                                thread_context=thread_context,
+                                                participant=chat_with_participant,
                                             )
                                         )
 
                                     await self.prepare_llm_context(
-                                        context=thread_context
+                                        thread_context=thread_context
                                     )
 
                                     llm_messages = asyncio.Queue[ResponseStreamEvent]()
@@ -752,12 +1021,32 @@ class ChatBot(SingleRoomAgent):
                                         )
                                     )
 
-                                    await self._llm_adapter.next(
-                                        context=chat_context,
-                                        room=self._room,
-                                        toolkits=thread_toolkits,
-                                        tool_adapter=self._tool_adapter,
+                                    message_toolkits = [*thread_toolkits]
+
+                                    model = received.message.get(
+                                        "model", self._llm_adapter.default_model()
+                                    )
+
+                                    message_tools = received.message.get("tools")
+
+                                    if (
+                                        message_tools is not None
+                                        and len(message_tools) > 0
+                                    ):
+                                        message_toolkits.extend(
+                                            make_tools(
+                                                model=model,
+                                                providers=thread_tool_providers,
+                                                tools=message_tools,
+                                            )
+                                        )
+
+                                    await self.handle_user_message(
+                                        context=thread_context,
+                                        toolkits=message_toolkits,
                                         event_handler=handle_event,
+                                        model=model,
+                                        from_user=chat_with_participant,
                                     )
 
                                     llm_messages.shutdown()
@@ -777,7 +1066,7 @@ class ChatBot(SingleRoomAgent):
                         finally:
 
                             async def cleanup():
-                                for participant in get_thread_participants(
+                                for participant in get_online_participants(
                                     room=self._room, thread=thread
                                 ):
                                     self._room.messaging.send_message_nowait(
@@ -817,6 +1106,48 @@ class ChatBot(SingleRoomAgent):
 
         self._thread_tasks.clear()
 
+    async def _on_get_thread_toolkits_message(self, *, message: RoomMessage):
+        path = message.message["path"]
+
+        thread_context = None
+        if path in self._open_threads:
+            thread = await self._open_threads[path]
+
+            thread_context = ChatThreadContext(
+                path=path,
+                chat=AgentChatContext(),
+                thread=thread,
+                participants=get_online_participants(room=self.room, thread=thread),
+            )
+
+        if thread_context is None:
+            logger.warning("thread toolkits requested for a thread that is not open")
+            return
+
+        chat_with_participant = None
+        for participant in self._room.messaging.get_participants():
+            if participant.id == message.from_participant_id:
+                chat_with_participant = participant
+                break
+
+        if chat_with_participant is None:
+            logger.warning(
+                "participant does not have messaging enabled, skipping message"
+            )
+            return
+
+        tool_providers = await self.get_thread_toolkit_builders(
+            thread_context=thread_context, participant=chat_with_participant
+        )
+        self._room.messaging.send_message_nowait(
+            to=chat_with_participant,
+            type="set_thread_tool_providers",
+            message={
+                "path": path,
+                "tool_providers": [{"name": t.name} for t in tool_providers],
+            },
+        )
+
     async def start(self, *, room):
         await super().start(room=room)
 
@@ -827,6 +1158,19 @@ class ChatBot(SingleRoomAgent):
         )
 
         def on_message(message: RoomMessage):
+            if message.type == "get_thread_toolkit_builders":
+                task = asyncio.create_task(
+                    self._on_get_thread_toolkits_message(message=message)
+                )
+
+                def on_done(task: asyncio.Task):
+                    try:
+                        task.result()
+                    except Exception as ex:
+                        logger.error(f"unable to get tool providers {ex}", exc_info=ex)
+
+                task.add_done_callback(on_done)
+
             if message.type == "chat" or message.type == "opened":
                 path = message.message["path"]