computer-use-ootb-internal 0.0.121__py3-none-any.whl → 0.0.123__py3-none-any.whl

computer_use_ootb_internal/computer_use_demo/animation/test_animation.py

@@ -1,40 +1,40 @@
-"""
-Test script to verify cursor animation is working
-"""
-import asyncio
-import sys
-import time
-from pathlib import Path
-from computer_use_ootb_internal.computer_use_demo.tools.computer import ComputerTool
-
-async def test_animations():
-    
-    # Initialize the computer tool
-    computer = ComputerTool()
-    
-    # Test mouse move animation
-    print("Testing mouse move animation...")
-    await computer(action="mouse_move_windll", coordinate=(500, 500))
-    print("Waiting 2 seconds...")
-    await asyncio.sleep(2)
-    
-    # Test click animation
-    print("Testing click animation...")
-    await computer(action="left_click_windll", coordinate=(700, 300))
-    print("Waiting 2 seconds...")
-    await asyncio.sleep(2)
-    
-    # Test another move
-    print("Testing move and click sequence...")
-    await computer(action="mouse_move_windll", coordinate=(300, 300))
-    await asyncio.sleep(1)
-    await computer(action="left_click_windll", coordinate=(300, 300))
-    
-    # Wait for animations to complete
-    print("Waiting for animations to complete...")
-    await asyncio.sleep(3)
-    
-    print("Test completed")
-
-if __name__ == "__main__":
+"""
+Test script to verify cursor animation is working
+"""
+import asyncio
+import sys
+import time
+from pathlib import Path
+from computer_use_ootb_internal.computer_use_demo.tools.computer import ComputerTool
+
+async def test_animations():
+    
+    # Initialize the computer tool
+    computer = ComputerTool()
+    
+    # Test mouse move animation
+    print("Testing mouse move animation...")
+    await computer(action="mouse_move_windll", coordinate=(500, 500))
+    print("Waiting 2 seconds...")
+    await asyncio.sleep(2)
+    
+    # Test click animation
+    print("Testing click animation...")
+    await computer(action="left_click_windll", coordinate=(700, 300))
+    print("Waiting 2 seconds...")
+    await asyncio.sleep(2)
+    
+    # Test another move
+    print("Testing move and click sequence...")
+    await computer(action="mouse_move_windll", coordinate=(300, 300))
+    await asyncio.sleep(1)
+    await computer(action="left_click_windll", coordinate=(300, 300))
+    
+    # Wait for animations to complete
+    print("Waiting for animations to complete...")
+    await asyncio.sleep(3)
+    
+    print("Test completed")
+
+if __name__ == "__main__":
     asyncio.run(test_animations()) 

computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py

@@ -92,12 +92,12 @@ class TeachmodeExecutor:
                 
                 if isinstance(tool_result, ToolResult):
                     print(f"[teachmode_executor] tool_result: {tool_result}")
-                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": "action", "action_type": tool_result.base_type}
+                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": tool_result.type, "action_type": tool_result.action_base_type}
                     yield tool_result_message
 
                 elif isinstance(tool_result, ToolError):
                     print(f"[teachmode_executor] tool_error: {tool_result}")
-                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": "error"}
+                    tool_result_message = {"role": "assistant", "content": tool_result.output, "type": "error", "action_type": ""}
                     yield tool_result_message
               
             return tool_result_message
@@ -197,7 +197,7 @@ class TeachmodeExecutor:
 
         except Exception as e:
             print(f"Error {e} in parsing output: {action_item}")
-            import pdb; pdb.set_trace()
+            # import pdb; pdb.set_trace()
             return None
 
 

computer_use_ootb_internal/computer_use_demo/tools/base.py

@@ -28,7 +28,7 @@ class ToolResult:
     error: str | None = None
     base64_image: str | None = None
     system: str | None = None
-    base_type: str | None = None
+    action_base_type: str | None = None
 
     def __bool__(self):
         return any(getattr(self, field.name) for field in fields(self))
@@ -66,6 +66,6 @@ class ToolFailure(ToolResult):
 class ToolError(Exception):
     """Raised when a tool encounters an error."""
 
-    def __init__(self, output: str, base_type: str):
+    def __init__(self, output: str, action_base_type: str):
         self.output = output
-        self.base_type = base_type
+        self.action_base_type = action_base_type

computer_use_ootb_internal/computer_use_demo/tools/computer.py

@@ -217,13 +217,13 @@ class ComputerTool(BaseAnthropicTool):
 
         if action in ("mouse_move", "left_click_drag"):
             if coordinate is None:
-                raise ToolError(output=f"coordinate is required for {action}", base_type="error")
+                raise ToolError(output=f"coordinate is required for {action}", action_base_type="error")
             if text is not None:
-                raise ToolError(output=f"text is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"text is not accepted for {action}", action_base_type="error")
             if not isinstance(coordinate, (list, tuple)) or len(coordinate) != 2:
-                raise ToolError(output=f"{coordinate} must be a tuple of length 2", base_type="error")
+                raise ToolError(output=f"{coordinate} must be a tuple of length 2", action_base_type="error")
             if not all(isinstance(i, int) for i in coordinate):
-                raise ToolError(output=f"{coordinate} must be a tuple of non-negative ints", base_type="error")
+                raise ToolError(output=f"{coordinate} must be a tuple of non-negative ints", action_base_type="error")
             
             if self.is_scaling:
                 x, y = self.scale_coordinates(
@@ -237,22 +237,22 @@ class ComputerTool(BaseAnthropicTool):
 
             if action == "mouse_move":
                 pyautogui.moveTo(x, y)
-                return ToolResult(output=f"Mouse move", base_type="move")
+                return ToolResult(output=f"Mouse move", action_base_type="move")
             
             elif action == "left_click_drag":
                 current_x, current_y = pyautogui.position()
                 pyautogui.dragTo(x, y, duration=0.5)  # Adjust duration as needed
-                return ToolResult(output=f"Mouse drag", base_type="move")
+                return ToolResult(output=f"Mouse drag", action_base_type="move")
 
         # Action Type 2: Required text (keynames)
         # Actions: key, type, key_down, key_up
         if action in ("key", "type", "key_down", "key_up"):
             if text is None:
-                raise ToolError(output=f"text is required for {action}", base_type="error")
+                raise ToolError(output=f"text is required for {action}", action_base_type="error")
             if coordinate is not None:
-                raise ToolError(output=f"coordinate is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"coordinate is not accepted for {action}", action_base_type="error")
             if not isinstance(text, str):
-                raise ToolError(output=f"{text} must be a string", base_type="error")
+                raise ToolError(output=f"{text} must be a string", action_base_type="error")
 
             if action == "key":
                 # Handle key combinations
@@ -265,19 +265,19 @@ class ComputerTool(BaseAnthropicTool):
                     key = self.key_conversion.get(key.strip(), key.strip())
                     key = key.lower()
                     pyautogui.keyUp(key)    # Release each key in reverse order
-                return ToolResult(output=f"Press key '{text}'", base_type="key")
+                return ToolResult(output=f"Press key '{text}'", action_base_type="key")
             
             elif action == "key_down":
                 pyautogui.keyDown(text)
-                return ToolResult(output=f"Press key '{text}'", base_type="key")
+                return ToolResult(output=f"Press key '{text}'", action_base_type="key")
             elif action == "key_up":
                 pyautogui.keyUp(text)
-                return ToolResult(output=f"Release key '{text}'", base_type="key")
+                return ToolResult(output=f"Release key '{text}'", action_base_type="key")
 
             elif action == "type":
                 pyautogui.typewrite(text, interval=TYPING_DELAY_MS / 1000)  # Convert ms to seconds
                 # screenshot_base64 = (await self.screenshot()).base64_image
-                return ToolResult(output=f"Type '{text}'", base_type="type") #  base64_image=screenshot_base64)
+                return ToolResult(output=f"Type '{text}'", action_base_type="type") #  base64_image=screenshot_base64)
 
         # Action Type 3: No required text or coordinates
         # Actions: left_click, right_click, double_click, middle_click, left_press, scroll_down, scroll_up
@@ -292,9 +292,9 @@ class ComputerTool(BaseAnthropicTool):
             "wait",
         ):
             if text is not None:
-                raise ToolError(output=f"text is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"text is not accepted for {action}", action_base_type="error")
             # if coordinate is not None:
-            #     raise ToolError(output=f"coordinate is not accepted for {action}", base_type="error")
+            #     raise ToolError(output=f"coordinate is not accepted for {action}", action_base_type="error")
 
             if coordinate is not None:
                 x, y = coordinate
@@ -304,70 +304,71 @@ class ComputerTool(BaseAnthropicTool):
             if action == "left_click":
                 show_click(x, y)
                 pyautogui.click(x=x, y=y)
-                return ToolResult(output="Left click", base_type="click")
+                return ToolResult(output="Left click", action_base_type="click")
             elif action == "right_click":
                 show_click(x, y)
                 pyautogui.rightClick(x=x, y=y)
-                return ToolResult(output="Right click", base_type="click")
+                return ToolResult(output="Right click", action_base_type="click")
             elif action == "middle_click":
                 show_click(x, y)
                 pyautogui.middleClick(x=x, y=y)
-                return ToolResult(output="Middle click", base_type="click")
+                return ToolResult(output="Middle click", action_base_type="click")
             elif action == "double_click":
                 show_click(x, y)
                 pyautogui.doubleClick(x=x, y=y)
-                return ToolResult(output="Double click", base_type="click")
+                return ToolResult(output="Double click", action_base_type="click")
             elif action == "left_press":
                 show_click(x, y)
                 pyautogui.mouseDown(x=x, y=y)
                 time.sleep(1)
                 pyautogui.mouseUp(x=x, y=y)
-                return ToolResult(output="Left press", base_type="click")
+                return ToolResult(output="Left press", action_base_type="click")
 
             elif action == "scroll_down":
                 pyautogui.scroll(-200)  # Adjust scroll amount as needed
-                return ToolResult(output="Scrolled down", base_type="scroll")
+                return ToolResult(output="Scrolled down", action_base_type="scroll")
             
             elif action == "scroll_up":
                 pyautogui.scroll(200)   # Adjust scroll amount as needed
-                return ToolResult(output="Scrolled up", base_type="scroll")
+                return ToolResult(output="Scrolled up", action_base_type="scroll")
             
             elif action == "wait":
                 time.sleep(15)
-                return ToolResult(output="Wait for next event", base_type="wait")
+                return ToolResult(output="Wait for next event", action_base_type="wait")
             
-            return ToolResult(output=f"Performed {action}", base_type="unknown")
+            return ToolResult(output=f"Performed {action}", action_base_type="unknown")
         
         # Action Type 4: Miscs. No required text or coordinates
         # Actions: screenshot, cursor_position
         if action in ("screenshot", "cursor_position"):
             if text is not None:
-                raise ToolError(output=f"text is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"text is not accepted for {action}", action_base_type="error")
             if coordinate is not None:
-                raise ToolError(output=f"coordinate is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"coordinate is not accepted for {action}", action_base_type="error")
+            
             if action == "screenshot":
                 return await self.screenshot()
             elif action == "cursor_position":
                 x, y = pyautogui.position()
                 # x, y = self.scale_coordinates(ScalingSource.COMPUTER, x, y)
-                return ToolResult(output=f"Cursor position ({x},{y})", base_type="unknown")
+                return ToolResult(output=f"Cursor position ({x},{y})", action_base_type="unknown")
 
         # Action Type 5: StarRail Mode
         # Actions: sr_scroll_down, sr_scroll_up
         if action in ("sr_scroll_down", "sr_scroll_up"):
             if text is not None:
-                raise ToolError(output=f"text is not accepted for {action}", base_type="error")
+                raise ToolError(output=f"text is not accepted for {action}", action_base_type="error")
 
             if action == "sr_scroll_down":
                 for _ in range(20):
                     pyautogui.scroll(-100)  # Adjust scroll amount as needed
                     time.sleep(0.001)
-                return ToolResult(output="Scroll down", base_type="scroll")
+                return ToolResult(output="Scroll down", action_base_type="scroll")
             elif action == "sr_scroll_up":
                 for _ in range(20):
                     pyautogui.scroll(100)   # Adjust scroll amount as needed
                     time.sleep(0.001)
-                return ToolResult(output="Scroll up", base_type="scroll")
+                return ToolResult(output="Scroll up", action_base_type="scroll")
 
         # starrail browser mode
         if action in ("left_click_windll", "mouse_move_windll", "right_click_windll", "key_down_windll", "key_up_windll"):
@@ -381,11 +382,11 @@ class ComputerTool(BaseAnthropicTool):
                     y = coordinate[1]+self.offset_y
                     show_click(x, y)
                     self.marbot_auto_gui.click(x=x, y=y)
-                return ToolResult(output=f"Left click", base_type="click")
+                return ToolResult(output=f"Left click", action_base_type="click")
 
             elif action == "mouse_move_windll":
                 if coordinate is None:
-                    raise ToolError(output=f"coordinate is required for {action}", base_type="error")
+                    raise ToolError(output=f"coordinate is required for {action}", action_base_type="error")
                 
                 x0, y0 = pyautogui.position()
                 # x0, y0 = self.scale_coordinates(ScalingSource.COMPUTER, x0, y0)
@@ -395,20 +396,21 @@ class ComputerTool(BaseAnthropicTool):
                 show_move_to(x0, y0, x1, y1)
                 self.marbot_auto_gui.moveTo(x=x1, y=y1)
             
-                return ToolResult(output=f"Mouse move", base_type="move")
+                return ToolResult(output=f"Mouse move", action_base_type="move")
             
             # elif action == "right_click_windll":
             #     self.marbot_auto_gui.rightClick(x=coordinate[0], y=coordinate[1])
             elif action == "key_down_windll":
                 self.marbot_auto_gui.keyDown(text)
-                return ToolResult(output=f"Key down '{text}'", base_type="key")
+                return ToolResult(output=f"Key down '{text}'", type="hidden", action_base_type="key")
+            
             elif action == "key_up_windll":
                 self.marbot_auto_gui.keyUp(text)
-                return ToolResult(output=f"Key up '{text}'", base_type="key")
+                return ToolResult(output=f"Key up '{text}'", type="hidden", action_base_type="key")
             
-            return ToolResult(output=f"Performed dll action:{action}", base_type="unknown")
+            return ToolResult(output=f"Performed dll action:{action}", action_base_type="unknown")
 
-        raise ToolError(output=f"Invalid action: {action}", base_type="error")
+        raise ToolError(output=f"Invalid action: {action}", type="hidden", action_base_type="error")
 
 
     async def screenshot(self):
@@ -499,9 +501,9 @@ class ComputerTool(BaseAnthropicTool):
 
         if path.exists():
             # Return a ToolResult instance instead of a dictionary
-            return ToolResult(base64_image=base64.b64encode(path.read_bytes()).decode(), base_type="screenshot")
+            return ToolResult(base64_image=base64.b64encode(path.read_bytes()).decode(), action_base_type="screenshot")
         
-        raise ToolError(output=f"Failed to take screenshot: {path} does not exist.", base_type="error")
+        raise ToolError(output=f"Failed to take screenshot: {path} does not exist.", action_base_type="error")
 
     def padding_image(self, screenshot):
         """Pad the screenshot to 16:10 aspect ratio, when the aspect ratio is not 16:10."""
@@ -551,7 +553,7 @@ class ComputerTool(BaseAnthropicTool):
         y_scaling_factor = target_dimension["height"] / self.height
         if source == ScalingSource.API:
             if x > self.width or y > self.height:
-                raise ToolError(output=f"Coordinates {x}, {y} are out of bounds", base_type="error")
+                raise ToolError(output=f"Coordinates {x}, {y} are out of bounds", action_base_type="error")
             # scale up
             return round(x / x_scaling_factor), round(y / y_scaling_factor)
         # scale down