meshagent-computers 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

meshagent/computers/__init__.py

@@ -5,3 +5,4 @@ from .docker import DockerComputer
 from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu
 from .operator import Operator
 from .agent import ComputerAgent
+from .version import __version__

meshagent/computers/agent.py

@@ -1,29 +1,182 @@
-from meshagent.openai import OpenAIResponsesAdapter
-from meshagent.agents import LLMAdapter, AgentChatContext
+from meshagent.agents import LLMAdapter
 from meshagent.tools import Tool, Toolkit, ToolContext
-from meshagent.agents.prompt import PromptAgent
-from meshagent.computers import Computer, Operator
+from meshagent.computers import Computer, Operator, BrowserbaseBrowser
 from meshagent.agents.chat import ChatBot, ChatThreadContext
-from meshagent.api import RemoteParticipant, FileResponse
+from meshagent.api import RequiredToolkit, RemoteParticipant
 from meshagent.api.messaging import RawOutputs
+from meshagent.tools.toolkit import register_toolkit_factory
 
-from typing import Optional
+from typing import Optional, Type, Callable
 import base64
-import json
 import logging
 
 logging.basicConfig()
 logger = logging.getLogger("computer")
 logger.setLevel(logging.INFO)
 
-class ComputerAgent[ComputerType:Computer, OperatorType:Operator](ChatBot):
+def make_computer_toolkit(*, operator_cls: Type[Operator], computer_cls: Type[Computer], render_screen: Callable[[bytes],None]):
+
+    operator = operator_cls()
+    computer = computer_cls()
+    started = False
+
+    class ComputerTool(Tool):
+        def __init__(self, *, operator: Operator, computer: Computer, title = "computer_call", description = "handle computer calls from computer use preview", rules = [], thumbnail_url = None, defs = None):
+            super().__init__(
+                name="computer_call",
+                # TODO: give a correct schema
+                input_schema={
+                    "additionalProperties" : False,
+                    "type" : "object",
+                    "required" : [],
+                    "properties" : {}
+                },
+                title=title,
+                description=description,
+                rules=rules,
+                thumbnail_url=thumbnail_url,
+                defs=defs,
+                
+            )
+            self.computer = computer
+
+
+        @property
+        def options(self):
+            return {
+                "type": "computer-preview",
+                "display_width": self.computer.dimensions[0],
+                "display_height": self.computer.dimensions[1],
+                "environment": self.computer.environment,
+            }
+
+        async def execute(self, context: ToolContext, *, arguments):
+            
+            nonlocal started
+            if started == False:
+                await self.computer.__aenter__()
+                started = True
+
+
+            outputs = await operator.play(computer=self.computer, item=arguments)
+            for output in outputs:
+                if output["type"] == "computer_call_output":
+                    if output["output"] != None:
+                        if output["output"]["type"] == "input_image":
+                        
+                            b64 : str = output["output"]["image_url"]
+                            image_data_b64 = b64.split(",", 1)
+                            
+                            image_bytes = base64.b64decode(image_data_b64[1])
+                            render_screen(image_bytes)
+                        
+                        
+            nonlocal computer_toolkit
+            if len(computer_toolkit.tools) == 1:
+                # HACK: after looking at the page, add the other tools,
+                # if we add these first then the computer-use-preview mode fails if it calls them before using the computer
+                computer_toolkit.tools.extend([
+                    ScreenshotTool(computer=computer),
+                    GotoURL(computer=computer),
+                ])
+            return RawOutputs(outputs=outputs)
+        
+    class ScreenshotTool(Tool):
+        def __init__(self, computer: Computer):
+            self.computer = computer
+
+            super().__init__(
+                name="screenshot",
+                # TODO: give a correct schema
+                input_schema={
+                    "additionalProperties" : False,
+                    "type" : "object",
+                    "required" : ["full_page","save_path"],
+                    "properties" : {
+                        "full_page" : {
+                            "type" : "boolean"
+                        },
+                        "save_path" : {
+                            "type" : "string",
+                            "description" : "a file path to save the screenshot to (should end with .png)"
+                        }
+                    }
+                },
+                description="take a screenshot of the current page",               
+            )
+
+        
+        async def execute(self, context: ToolContext, save_path: str, full_page: bool):
+            nonlocal started
+            if started == False:
+                await self.computer.__aenter__()
+                started = True
+
+            screenshot_bytes = await self.computer.screenshot_bytes(full_page=full_page)
+            handle = await context.room.storage.open(path=save_path, overwrite=True)
+            await context.room.storage.write(handle=handle, data=screenshot_bytes)
+            await context.room.storage.close(handle=handle)
+
+            return f"saved screenshot to {save_path}"
+        
+    class GotoURL(Tool):
+        def __init__(self, computer: Computer):
+            self.computer = computer
+
+            super().__init__(
+                name="goto",
+                description="goes to a specific URL. Make sure it starts with http:// or https://",
+                # TODO: give a correct schema
+                input_schema={
+                    "additionalProperties" : False,
+                    "type" : "object",
+                    "required" : ["url"],
+                    "properties" : {
+                        "url" : {
+                            "type" : "string",
+                            "description": "Fully qualified URL to navigate to.",
+                        }
+                    }
+                },
+            )
+
+        
+        async def execute(self, context: ToolContext, url: str):
+            nonlocal started
+            if started == False:
+                await self.computer.__aenter__()
+                started = True
+
+            if url.startswith("https://") == False and url.startswith("http://") == False:
+                url = "https://"+url
+
+            await self.computer.goto(url)
+
+            render_screen(await self.computer.screenshot_bytes(full_page=False))
+
+
+    computer_tool = ComputerTool(computer=computer, operator=operator)
+
+    computer_toolkit = Toolkit(name="meshagent.openai.computer", tools=[
+        computer_tool
+    ])
+
+    return computer_toolkit
+
+async def make_browserbase_toolkit(context: ToolContext, requirement: RequiredToolkit):
+    return make_computer_toolkit(operator_cls=Operator(), computer_cls=BrowserbaseBrowser())
+
+register_toolkit_factory("browserbase", make_browserbase_toolkit)
+
+
+class ComputerAgent(ChatBot):
     def __init__(self, *, name, 
             title=None,
             description=None,
             requires=None,
             labels = None,
-            computer_cls: ComputerType,
-            operator_cls: OperatorType,
+            computer_cls: Type[Computer],
+            operator_cls: Type[Operator],
             rules: Optional[list[str]] = None,
             llm_adapter: Optional[LLMAdapter] = None,
             toolkits: list[Toolkit] = None
@@ -48,178 +201,28 @@ class ComputerAgent[ComputerType:Computer, OperatorType:Operator](ChatBot):
         self.operator_cls = operator_cls
 
 
-    async def init_thread_context(self, *, thread_context: ChatThreadContext):
+    async def get_thread_toolkits(self, *, thread_context: ChatThreadContext, participant: RemoteParticipant):
         
-        operator : Operator = self.operator_cls()
-        computer : Computer = self.computer_cls()
-        started = False
-
-        class ComputerTool(Tool):
-            def __init__(self, *, operator: Operator, computer: Computer, title = "computer_call", description = "handle computer calls from computer use preview", rules = [], thumbnail_url = None, defs = None):
-                super().__init__(
-                    name="computer_call",
-                    # TODO: give a correct schema
-                    input_schema={
-                        "additionalProperties" : False,
-                        "type" : "object",
-                        "required" : [],
-                        "properties" : {}
-                    },
-                    title=title,
-                    description=description,
-                    rules=rules,
-                    thumbnail_url=thumbnail_url,
-                    defs=defs,
-                   
-                )
-                self.computer = computer
-
-
-            @property
-            def options(self):
-                return {
-                    "type": "computer-preview",
-                    "display_width": self.computer.dimensions[0],
-                    "display_height": self.computer.dimensions[1],
-                    "environment": self.computer.environment,
-                }
-
-            async def execute(self,  context: ToolContext, *, arguments):
-                
-                nonlocal started
-                if started == False:
-                    await self.computer.__aenter__()
-                    started = True
-
-                for participant in thread_context.participants:
-                    await context.room.messaging.send_message(
-                        to=participant,
-                        type="computer_use",
-                        message={
-                            "arguments" : arguments
-                        }
-                    )
-
-                outputs = await operator.play(computer=self.computer, item=arguments)
-                for output in outputs:
-                      if output["type"] == "computer_call_output":
-                          if output["output"] != None:
-                              if output["output"]["type"] == "input_image":
-                                  
-                                b64 : str = output["output"]["image_url"]
-                                image_data_b64 = b64.split(",", 1)
-                                
-                                image_bytes = base64.b64decode(image_data_b64[1])
-
-                                for participant in thread_context.participants:
-                                    context.room.messaging.send_message_nowait(
-                                        to=participant,
-                                        type="computer_screen",
-                                        message={
-                                        },
-                                        attachment=image_bytes
-                                    )
-                
-                nonlocal computer_toolkit
-                if len(computer_toolkit.tools) == 1:
-                    # HACK: after looking at the page, add the other tools,
-                    # if we add these first then the computer-use-preview mode fails if it calls them before using the computer
-                    computer_toolkit.tools.extend([
-                        ScreenshotTool(computer=computer),
-                        GotoURL(computer=computer),
-                    ])
-                return RawOutputs(outputs=outputs)
-            
-        class ScreenshotTool(Tool):
-            def __init__(self, computer: Computer):
-                self.computer = computer
-
-                super().__init__(
-                    name="screenshot",
-                    # TODO: give a correct schema
-                    input_schema={
-                        "additionalProperties" : False,
-                        "type" : "object",
-                        "required" : ["full_page","save_path"],
-                        "properties" : {
-                            "full_page" : {
-                                "type" : "boolean"
-                            },
-                            "save_path" : {
-                                "type" : "string",
-                                "description" : "a file path to save the screenshot to (should end with .png)"
-                            }
-                        }
+        toolkits = await super().get_thread_toolkits(thread_context=thread_context, participant=participant)
+
+        def render_screen(image_bytes: bytes):
+            for participant in thread_context.participants:
+                self.room.messaging.send_message_nowait(
+                    to=participant,
+                    type="computer_screen",
+                    message={
                     },
-                    description="take a screenshot of the current page",               
+                    attachment=image_bytes
                 )
-
             
-            async def execute(self, context: ToolContext, save_path: str, full_page: bool):
-                nonlocal started
-                if started == False:
-                    await self.computer.__aenter__()
-                    started = True
-
-                screenshot_bytes = await self.computer.screenshot_bytes(full_page=full_page)
-                handle = await context.room.storage.open(path=save_path, overwrite=True)
-                await context.room.storage.write(handle=handle, data=screenshot_bytes)
-                await context.room.storage.close(handle=handle)
-
-                return f"saved screenshot to {save_path}"
-            
-        class GotoURL(Tool):
-            def __init__(self, computer: Computer):
-                self.computer = computer
-
-                super().__init__(
-                    name="goto",
-                    description="goes to a specific URL. Make sure it starts with http:// or https://",
-                    # TODO: give a correct schema
-                    input_schema={
-                        "additionalProperties" : False,
-                        "type" : "object",
-                        "required" : ["url"],
-                        "properties" : {
-                            "url" : {
-                                "type" : "string",
-                                "description": "Fully qualified URL to navigate to.",
-                            }
-                        }
-                    },
-                )
-
-            
-            async def execute(self, context: ToolContext, url: str):
-                nonlocal started
-                if started == False:
-                    await self.computer.__aenter__()
-                    started = True
-
-                if url.startswith("https://") == False and url.startswith("http://") == False:
-                    url = "https://"+url
-
-                await self.computer.goto(url)
-
-                # send an updated screen out
-                for participant in thread_context.participants:
-                    context.room.messaging.send_message_nowait(
-                        to=participant,
-                        type="computer_screen",
-                        message={
-                        },
-                        attachment = await self.computer.screenshot_bytes(full_page=False)
-                    )
-        
-        computer_tool = ComputerTool(computer=computer, operator=operator)
-        
-        computer_toolkit = Toolkit(name="meshagent.openai.computer", tools=[
-            computer_tool
-        ])
-
-        thread_context.toolkits = [
+        computer_toolkit = make_computer_toolkit(
+            operator_cls=self.operator_cls,
+            computer_cls=self.computer_cls,
+            render_screen=render_screen
+        )
+        return [
             computer_toolkit,
-            *thread_context.toolkits
+            *toolkits
         ]
 
 

meshagent/computers/utils.py

@@ -47,7 +47,7 @@ def sanitize_message(msg: dict) -> dict:
     return msg
 
 
-def create_response(**kwargs):
+def track_response(**kwargs):
     url = "https://api.openai.com/v1/responses"
     headers = {
         "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",

meshagent/computers/version.py

@@ -0,0 +1 @@
+__version__ = "0.0.8"

{meshagent_computers-0.0.7.dist-info → meshagent_computers-0.0.8.dist-info}/METADATA

@@ -1,9 +1,8 @@
 Metadata-Version: 2.4
 Name: meshagent-computers
-Version: 0.0.7
+Version: 0.0.8
 Summary: Computer Building Blocks for Meshagent
-Home-page: 
-License: Apache License 2.0
+License-Expression: Apache-2.0
 Project-URL: Documentation, https://meshagent.com
 Project-URL: Website, https://meshagent.com
 Project-URL: Source, https://github.com/meshagent
@@ -13,16 +12,10 @@ License-File: LICENSE
 Requires-Dist: pytest>=8.3.4
 Requires-Dist: pytest-asyncio>=0.24.0
 Requires-Dist: openai>=1.66.2
-Requires-Dist: meshagent-api>=0.0.7
-Requires-Dist: meshagent-agents>=0.0.7
-Requires-Dist: meshagent-tools>=0.0.7
+Requires-Dist: meshagent-api>=0.0.8
+Requires-Dist: meshagent-agents>=0.0.8
+Requires-Dist: meshagent-tools>=0.0.8
 Requires-Dist: playwright~=1.50.0
 Requires-Dist: browserbase~=1.2.0
 Requires-Dist: scrapybara~=2.4.1
-Dynamic: description-content-type
-Dynamic: license
 Dynamic: license-file
-Dynamic: project-url
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary

{meshagent_computers-0.0.7.dist-info → meshagent_computers-0.0.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
-meshagent/computers/__init__.py,sha256=4aVM46U1kVLeN8rurNulc6_24a-rHSMIY0v0rc1ydOI,288
-meshagent/computers/agent.py,sha256=w9Yg9I2cfx-fNo9Y0TOcndxcoyOTz2wxRoijxGt9dB4,9089
+meshagent/computers/__init__.py,sha256=PEzgkWE_sB6TrBA8crwWM7QaNPYWJisCKT81tk6pepQ,321
+meshagent/computers/agent.py,sha256=AR1MDxuz7DiLul8H5aluU5z61Yt7lGqmDo6l1QKWRwA,8459
 meshagent/computers/base_playwright.py,sha256=NhQoOzWmekBmobQ2kvN7w4I3hZeyMIuMKI2eHbG1_mk,6017
 meshagent/computers/browserbase.py,sha256=ZT-ZDndkBD-MHD3UjxXpwpx7-_VCkkanjW1rIJr7-98,8161
 meshagent/computers/computer.py,sha256=465GKDV3cJgzp1SB5B0xzLoftMKU0AxAuBVrZ9drBBc,1072
@@ -7,9 +7,10 @@ meshagent/computers/docker.py,sha256=BGkrlspIYlg3WViaqctyRUye-fONfF40AvLXOQuXPQ8
 meshagent/computers/local_playwright.py,sha256=W7VhxIYEXn2BO3wZZVsc-hH-CU7NTssxFGO7pwz_n1I,919
 meshagent/computers/operator.py,sha256=JTyGvoPnKOqUDaZv58p_2n5Q6m6dm8V214VMIhVEu08,2913
 meshagent/computers/scrapybara.py,sha256=IoeV02QcBDkpW05A1NH-lc4jnBEmZ4SPsRo3nnXC3UE,7502
-meshagent/computers/utils.py,sha256=t9G9dzfJ5a0DSyrW1KNdzlhLJW_S6eGUcS39wnZyHU8,2081
-meshagent_computers-0.0.7.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
-meshagent_computers-0.0.7.dist-info/METADATA,sha256=eYIzYXp2Ia_WFN1mjBrF7u9iPk08P-MZLifVB56PqRM,852
-meshagent_computers-0.0.7.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-meshagent_computers-0.0.7.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
-meshagent_computers-0.0.7.dist-info/RECORD,,
+meshagent/computers/utils.py,sha256=N7Ltjx3HfhJ-CYectNodGsBtddk6mSTHkCH0meqHT70,2080
+meshagent/computers/version.py,sha256=ICOxGt7ta-t2F9uXLMXVo-4VDYGM0IEvKPStCf0y8Rk,21
+meshagent_computers-0.0.8.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_computers-0.0.8.dist-info/METADATA,sha256=24jshdqcrgglfb-LQfoFjt2wLgqEmiOKwvcPJW8jEG8,706
+meshagent_computers-0.0.8.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+meshagent_computers-0.0.8.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_computers-0.0.8.dist-info/RECORD,,

{meshagent_computers-0.0.7.dist-info → meshagent_computers-0.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any