computer-use-ootb-internal 0.0.109__py3-none-any.whl → 0.0.109.post2__py3-none-any.whl

computer_use_ootb_internal/app_teachmode.py

@@ -5,14 +5,12 @@ from datetime import datetime
 import threading
 import requests
 import platform  # Add platform import
-import subprocess  # Add subprocess import
 import pyautogui  # Add pyautogui import
 import webbrowser # Add webbrowser import
 import os # Import os for path joining
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
-from screeninfo import get_monitors
 from computer_use_ootb_internal.computer_use_demo.tools.computer import get_screen_details
 from computer_use_ootb_internal.run_teachmode_ootb_args import simple_teachmode_sampling_loop
 

computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py

@@ -92,11 +92,12 @@ class TeachmodeExecutor:
                 
                 if isinstance(tool_result, ToolResult):
                     print(f"[teachmode_executor] tool_result: {tool_result}")
-                    tool_result_message = {"role": "assistant", "content": tool_result['output'], "type": "action", "action_type": tool_result['base_type']}
+                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": "action", "action_type": tool_result.base_type}
                     yield tool_result_message
 
                 elif isinstance(tool_result, ToolError):
-                    tool_result_message = {"role": "assistant", "content": tool_result['output'], "type": "error"}
+                    print(f"[teachmode_executor] tool_error: {tool_result}")
+                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": "error"}
                     yield tool_result_message
               
             return tool_result_message
@@ -134,7 +135,7 @@ class TeachmodeExecutor:
                 x, y = action_item["position"]
                 action_item["position"] = (int(x), int(y))
                 refined_output.append({"action": "mouse_move", "text": None, "coordinate": tuple(action_item["position"])})
-                refined_output.append({"action": "left_click", "text": None, "coordinate": None})
+                refined_output.append({"action": "left_click", "text": None, "coordinate": tuple(action_item["position"])})
             
             elif action_item["action"] == "INPUT":  # 2. input -> type
                 if "text" in action_item:

computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py

@@ -4,6 +4,13 @@ import datetime
 import numpy as np
 import cv2
 
+def get_screen_resize_factor():
+     # import ctypes
+     # scaleFactor = ctypes.windll.shcore.GetScaleFactorForDevice(0) / 100
+     # scaleFactor = str(scaleFactor) + "x"
+     # return scaleFactor
+     return "1.0x"
+
 
 def multivalue_image(img, mode='None', thresholds=None, interval_values=None, save=True, cache_folder=None):
     img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

computer_use_ootb_internal/computer_use_demo/tools/computer.py

@@ -293,36 +293,37 @@ class ComputerTool(BaseAnthropicTool):
         ):
             if text is not None:
                 raise ToolError(output=f"text is not accepted for {action}", base_type="error")
+            # if coordinate is not None:
+            #     raise ToolError(output=f"coordinate is not accepted for {action}", base_type="error")
+
             if coordinate is not None:
-                raise ToolError(output=f"coordinate is not accepted for {action}", base_type="error")
+                x, y = coordinate
+            else:
+                x, y = pyautogui.position()
 
             if action == "left_click":
-                x, y = pyautogui.position()
                 show_click(x, y)
-                pyautogui.click()
+                pyautogui.click(x=x, y=y)
                 return ToolResult(output="Left click", base_type="click")
             elif action == "right_click":
-                x, y = pyautogui.position()
                 show_click(x, y)
-                pyautogui.rightClick()
+                pyautogui.rightClick(x=x, y=y)
                 return ToolResult(output="Right click", base_type="click")
             elif action == "middle_click":
-                x, y = pyautogui.position()
                 show_click(x, y)
-                pyautogui.middleClick()
+                pyautogui.middleClick(x=x, y=y)
                 return ToolResult(output="Middle click", base_type="click")
             elif action == "double_click":
-                x, y = pyautogui.position()
                 show_click(x, y)
-                pyautogui.doubleClick()
+                pyautogui.doubleClick(x=x, y=y)
                 return ToolResult(output="Double click", base_type="click")
             elif action == "left_press":
-                x, y = pyautogui.position()
                 show_click(x, y)
-                pyautogui.mouseDown()
+                pyautogui.mouseDown(x=x, y=y)
                 time.sleep(1)
-                pyautogui.mouseUp()
+                pyautogui.mouseUp(x=x, y=y)
                 return ToolResult(output="Left press", base_type="click")
+
             elif action == "scroll_down":
                 pyautogui.scroll(-200)  # Adjust scroll amount as needed
                 return ToolResult(output="Scrolled down", base_type="scroll")
@@ -391,7 +392,7 @@ class ComputerTool(BaseAnthropicTool):
                 x1 = coordinate[0]+self.offset_x
                 y1 = coordinate[1]+self.offset_y
 
-                show_move_to(x0, y0, x1, y1, duration_ms=1000)
+                show_move_to(x0, y0, x1, y1)
                 self.marbot_auto_gui.moveTo(x=x1, y=y1)
             
                 return ToolResult(output=f"Mouse move", base_type="move")

computer_use_ootb_internal/run_teachmode_ootb_args.py

@@ -7,9 +7,8 @@ import datetime
 from datetime import datetime, timedelta, timezone
 
 from computer_use_ootb_internal.computer_use_demo.executor.teachmode_executor import TeachmodeExecutor
-from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.gui_capture import get_screenshot
-from computer_use_ootb_internal.computer_use_demo.gui_agent.llm_utils.llm_utils import encode_image, is_image_path
-from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.icon_detection.icon_detection import get_screen_resize_factor
+from computer_use_ootb_internal.computer_use_demo.gui_agent.llm_utils.llm_utils import is_image_path
+from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.utils import get_screen_resize_factor
 from computer_use_ootb_internal.computer_use_demo.tools.aws_request import send_request_to_server
 from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.uia_tools.screenshot_service import get_screenshot_external_cmd
 utc_plus_8 = timezone(timedelta(hours=8))

{computer_use_ootb_internal-0.0.109.dist-info → computer_use_ootb_internal-0.0.109.post2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: computer-use-ootb-internal
-Version: 0.0.109
+Version: 0.0.109.post2
 Summary: Computer Use OOTB
 Author-email: Siyuan Hu <siyuan.hu.sg@gmail.com>
 Requires-Python: >=3.11

{computer_use_ootb_internal-0.0.109.dist-info → computer_use_ootb_internal-0.0.109.post2.dist-info}/RECORD

@@ -1,16 +1,16 @@
 computer_use_ootb_internal/README.md,sha256=FxpW95lyub2iX73ZDfK6ML7SdEKg060H5I6Grub7li4,31
-computer_use_ootb_internal/app_teachmode.py,sha256=sO_li1RAUlqCdXF5oDenGopgCKuAwAlevSISy6uCvkk,19334
+computer_use_ootb_internal/app_teachmode.py,sha256=Yb5LtyaW4agGsfTmVvnIAkERXILkg3KkUZf7yI1dW-g,19253
 computer_use_ootb_internal/app_teachmode_gradio.py,sha256=cmFpBrkdlZxOQADWveVdIaaNqaBD8IVs-xNLJogU7F8,7909
 computer_use_ootb_internal/dependency_check.py,sha256=y8RMEP6RXQzTgU1MS_1piBLtz4J-Hfn9RjUZg59dyvo,1333
 computer_use_ootb_internal/requirements-lite.txt,sha256=5DAHomz4A_P2BmTIXNkNqkHbnIF0AyZ4_1XAlb1LaYs,290
-computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=vEgsnlnpdggobJYvi7ktbTH8UqT6hNnxBfhjf6dgxzE,7243
+computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=7Dj0iY4GG7P03tRKYJ2x9Yvt-PE-b7uyjCAed3SaF3Y,7086
 computer_use_ootb_internal/computer_use_demo/animation/click_animation.py,sha256=tP1gsayFy-CKk10UlrE9RlexwlHWiHQUe5Ogg4vQvSg,3234
 computer_use_ootb_internal/computer_use_demo/animation/icons8-select-cursor-transparent-96.gif,sha256=4LfwsfFQnREXrNRs32aJU2jO65JXianJoL_8q7-8elg,30966
 computer_use_ootb_internal/computer_use_demo/animation/test_animation.py,sha256=SOJz2yffXTkjuAHqk0IZLcMriR0KQYTo7W1b8wGyRGY,1222
-computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=1xver_8cRCrFk85OYqEQ7LUPYs5v2Mvtg944wiGN05A,16562
+computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=TBp_rXSAuHFBmZYyyIFca404c4v5LG1ZVbNaQpuvtzQ,16656
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/__init__.py,sha256=h2CNeuACklxVpJC65QR8_6AvSybEZLmeO45hY_-lLBs,61
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/gui_capture.py,sha256=CxFJbsSb68ERKH7-C4RaaZy7FIhhzrzGx5qQJ4C37cA,13907
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py,sha256=GEA1ES7vOpHBg_Suxpl99reh34kRG4RQpp072JQBK5c,9787
+computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py,sha256=OOVxy4Qlbk5q-X9kXFXqt6AmuOMl6FWWqtH269DvJJA,10005
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/screenshot_cli.py,sha256=hrUBH3aeUe9t4EDsMhaUZLPpNLKeEaG4ZvikrFy-bOU,1313
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/screenshot_service.py,sha256=fJ8G4_-ZLGsHLdLgHCNgjVM1osbfz21L2w8QrcvnuhA,2738
@@ -18,20 +18,18 @@ computer_use_ootb_internal/computer_use_demo/gui_agent/llm_utils/llm_utils.py,sh
 computer_use_ootb_internal/computer_use_demo/gui_agent/llm_utils/oai.py,sha256=zWAHa37fhIWCYpIEvQsIe5prfBJQ9yZFQwAi_dm8baE,4158
 computer_use_ootb_internal/computer_use_demo/gui_agent/llm_utils/run_litellm.py,sha256=SBnWMhwZoCFji9-ii7VSfPYAhDKUWvlJgJzw9-Ei3-g,7750
 computer_use_ootb_internal/computer_use_demo/gui_agent/llm_utils/run_llm.py,sha256=fxC-7lg8TLAi9f69zs5y5Pwga8Y5mY7Yfc5NNZNRJgM,1558
-computer_use_ootb_internal/computer_use_demo/gui_agent/vlm_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-computer_use_ootb_internal/computer_use_demo/gui_agent/vlm_utils/run_vlm.py,sha256=9IaX15dhFFOALr0vcsI-EtEU4pDLfdzlmwCy3HRbISA,1441
 computer_use_ootb_internal/computer_use_demo/tools/__init__.py,sha256=Pj8_5L4_PPQK298X4NV3KMbP-84t-bM0pbjEeb5_SJQ,343
 computer_use_ootb_internal/computer_use_demo/tools/aws_request.py,sha256=12UVzeA2PmpZhpy2Pt5Vh48-_q1e8ZjVQux6r3pbAcw,2629
 computer_use_ootb_internal/computer_use_demo/tools/base.py,sha256=4HHSxDg6GsCoraamWor75ZnoR7tAcRarMv4MzvqMWGU,2099
 computer_use_ootb_internal/computer_use_demo/tools/bash.py,sha256=rHetQ80_v-TTi-1oxIA7ncFEwJxFTh8FJCErIoZbGeY,4236
 computer_use_ootb_internal/computer_use_demo/tools/collection.py,sha256=8RzHLobL44_Jjt8ltXS6I8XJlEAQOfc75dmnDUaHE-8,922
 computer_use_ootb_internal/computer_use_demo/tools/colorful_text.py,sha256=cvlmnhAImDTwoRRwhT5au7mNFhfAD7ZfeoDEVdVzDKw,892
-computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=Z-kt6XxGIeZJaeaj02ktAcUQL8vG7co2bedzC5pSAsc,26794
+computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=lm3bzh-ZzeFGAqrgQGkDr6SnvoB9d4NvVo8D1PNOoAk,26745
 computer_use_ootb_internal/computer_use_demo/tools/computer_marbot.py,sha256=zZuWz9ArfP3Zss-afnscrPkgCtB5UWbCy7HwAOvO2bo,5970
 computer_use_ootb_internal/computer_use_demo/tools/edit.py,sha256=b0PwUitxckHCQqFP3ZwlthWdqNkn7WETeTHeB6-o98c,11486
 computer_use_ootb_internal/computer_use_demo/tools/run.py,sha256=xhXdnBK1di9muaO44CEirL9hpGy3NmKbjfMpyeVmn8Y,1595
 computer_use_ootb_internal/computer_use_demo/tools/screen_capture.py,sha256=L8qfvtUkPPQGt92N-2Zfw5ZTDBzLsDps39uMnX3_uSA,6857
-computer_use_ootb_internal-0.0.109.dist-info/METADATA,sha256=d9XQc6D4WXNxoMB2p8Zs_T6AwBHjO6T6sxf1NYHS4mU,910
-computer_use_ootb_internal-0.0.109.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-computer_use_ootb_internal-0.0.109.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
-computer_use_ootb_internal-0.0.109.dist-info/RECORD,,
+computer_use_ootb_internal-0.0.109.post2.dist-info/METADATA,sha256=jTj_Um-WzpFJjLXIQ_iK6veWsvkwPcuvJ5p4GTC0vCA,916
+computer_use_ootb_internal-0.0.109.post2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+computer_use_ootb_internal-0.0.109.post2.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
+computer_use_ootb_internal-0.0.109.post2.dist-info/RECORD,,

computer_use_ootb_internal/computer_use_demo/gui_agent/vlm_utils/run_vlm.py

@@ -1,46 +0,0 @@
-import base64
-import logging
-
-
-def run_vlm(messages, system, vlm="gpt-4o-mini", max_tokens=512, temperature=0, stop=None):
-    log_prompt(messages)
-    
-    # turn string prompt into list
-    if isinstance(messages, str):
-        messages = [messages]
-    elif isinstance(messages, list):
-        pass
-    else:
-        raise ValueError(f"Invalid prompt type: {type(messages)}")
-    
-    if vlm.startswith("gpt"): # gpt series
-        from .oai import run_oai_interleaved
-        response, token_usage = run_oai_interleaved(
-            messages=messages, 
-            system=system, 
-            llm=llm, 
-            max_tokens=max_tokens, 
-            temperature=temperature
-        )
-    elif vlm.startswith("gemini"): # gemini series
-        from .gemini import run_gemini_interleaved
-
-        response, token_usage = run_gemini_interleaved(
-            messages=messages, 
-            system=system, 
-            vlm=vlm, 
-            max_tokens=max_tokens, 
-            temperature=temperature
-        )
-    else:
-        raise ValueError(f"Invalid llm: {llm}")
-    logging.info(
-        f"========Output for {vlm}=======\n{response}\n============================")
-    return response
-
-def log_prompt(prompt):
-    prompt_display = [prompt] if isinstance(prompt, str) else prompt
-    prompt_display = "\n\n".join(prompt_display)
-    logging.info(
-        f"========Prompt=======\n{prompt_display}\n============================")
-