computer-use-ootb-internal 0.0.103__py3-none-any.whl → 0.0.104__py3-none-any.whl

computer_use_ootb_internal/app_teachmode.py

@@ -4,6 +4,11 @@ import json
 from datetime import datetime
 import threading
 import requests
+import platform  # Add platform import
+import subprocess  # Add subprocess import
+import pyautogui  # Add pyautogui import
+import webbrowser # Add webbrowser import
+import os # Import os for path joining
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
@@ -87,6 +92,99 @@ class SharedState:
 shared_state = None
 rate_limiter = RateLimiter(interval_seconds=2)
 
+# Add the new prepare_environment function here
+def prepare_environment(state):
+    """Prepares the environment before starting the main processing loop, e.g., opening specific apps."""
+    if platform.system() == "Windows":
+        # Assuming Star Rail mode is indicated by user_id containing "star_rail"
+        # You might need to adjust this condition based on the actual logic in run_teachmode_args
+        is_star_rail = "star_rail" in state.user_id.lower() or \
+                       "star_rail" in state.trace_id.lower() or \
+                       "hero_case" in state.trace_id.lower()
+        
+        if is_star_rail:
+            print("Star Rail mode detected on Windows. Opening Edge browser...")
+            url = "https://sr.mihoyo.com/cloud/#/"
+            browser_opened = False
+            try:
+                print(f"Attempting to open {url} using specific Edge paths...")
+                # Common paths for msedge.exe
+                edge_paths = [
+                    os.path.join(os.environ.get("ProgramFiles(x86)", "C:\Program Files (x86)"), "Microsoft\Edge\Application\msedge.exe"),
+                    os.path.join(os.environ.get("ProgramFiles", "C:\Program Files"), "Microsoft\Edge\Application\msedge.exe")
+                ]
+                
+                opened_with_subprocess = False
+                for edge_path in edge_paths:
+                    if os.path.exists(edge_path):
+                        try:
+                            print(f"Trying path: {edge_path}")
+                            subprocess.Popen([edge_path, url])
+                            print(f"Successfully launched Edge with Popen using path: {edge_path}")
+                            opened_with_subprocess = True
+                            browser_opened = True
+                            break # Exit loop once successfully opened
+                        except Exception as sub_e:
+                            print(f"Failed to launch Edge with Popen using {edge_path}: {sub_e}")
+                    else:
+                        print(f"Edge path not found: {edge_path}")
+
+                # Fallback to webbrowser.open if subprocess failed
+                if not opened_with_subprocess:
+                    print("Subprocess launch failed, falling back to webbrowser.open()...")
+                    try:
+                         if webbrowser.open(url):
+                             print("Successfully opened browser using webbrowser.open() as fallback.")
+                             browser_opened = True
+                         else:
+                             print("webbrowser.open() also failed to indicate success.")
+                    except Exception as web_e:
+                         print(f"webbrowser.open() fallback failed: {web_e}")
+
+                if not browser_opened:
+                    print("ERROR: Failed to open browser using both subprocess and webbrowser.")
+                    # Decide if you want to proceed without the browser or raise an error
+                    # For now, we'll proceed to the click attempt but it might fail
+
+                # Add pyautogui click after attempting to open the browser
+                print("Proceeding with pyautogui actions...")
+                time.sleep(5) # Wait time for the browser to load
+                
+                # Print detected screen size
+                screen_width, screen_height = pyautogui.size()
+                print(f"Detected screen size: {screen_width}x{screen_height}")
+
+                click_x = int(screen_width * (1036 / 1280))
+                click_y = int(screen_height * (500 / 720))
+                print(f"Calculated click coordinates: ({click_x}, {click_y})")
+
+                # Disable failsafe before clicking
+                pyautogui.FAILSAFE = False
+                print("PyAutoGUI failsafe temporarily disabled.")
+
+                print(f"Clicking at coordinates: ({click_x}, {click_y})")
+                pyautogui.click(click_x, click_y)
+                time.sleep(2)
+                pyautogui.click(click_x, click_y)
+                
+                # Re-enable failsafe (optional, as script might end anyway)
+                # pyautogui.FAILSAFE = True 
+                # print("PyAutoGUI failsafe re-enabled.")
+
+            except FileNotFoundError:
+                # This specific error might occur if edge_path exists but execution fails
+                print("Error: Could not execute the browser command (FileNotFound). Check permissions or path.")
+            except Exception as e:
+                print(f"Error during environment preparation (browser/click): {e}")
+            finally:
+                 # Ensure failsafe is re-enabled if an error occurs after disabling it
+                 pyautogui.FAILSAFE = True
+                 print("PyAutoGUI failsafe re-enabled.")
+    else:
+        # Placeholder for potential preparations on other OS or non-Star Rail modes
+        print("Environment preparation: No specific actions required for this OS/mode.")
+
+
 @app.post("/update_params")
 async def update_parameters(request: Request):
     data = await request.json()
@@ -111,6 +209,9 @@ async def update_parameters(request: Request):
     
     log_ootb_request(shared_state.server_url, "update_params", data)
 
+    # Call the preparation function here, after parameters are updated
+    prepare_environment(shared_state)
+
     return JSONResponse(
         content={"status": "success", "message": "Parameters updated", "new_args": vars(shared_state.args)},
         status_code=200

computer_use_ootb_internal/computer_use_demo/animation/click_animation.py

@@ -1,81 +1,214 @@
-# click_anim_async.py   ←  put this in its own file  (important for Windows "spawn")
-import sys, multiprocessing as mp
+"""
+show_click(x, y, duration_ms=800)
+ → 在屏幕 (x,y) 显示点击动画，停留 duration_ms 毫秒
+依赖: pyside6
+确保同目录有 click.gif
+"""
+import sys, time
 from pathlib import Path
-from PySide6.QtCore    import Qt, QPoint, QTimer, QEasingCurve, QPropertyAnimation, QSize
-from PySide6.QtGui     import QMovie
+from PySide6.QtCore import Qt, QPoint, QTimer, QEventLoop, QSize, QEasingCurve, QPropertyAnimation
+from PySide6.QtGui  import QPainter, QPixmap, QMovie
 from PySide6.QtWidgets import QApplication, QWidget, QLabel
 
 CLICK_GIF = Path(__file__).with_name("icons8-select-cursor-transparent-96.gif")
 
-# ---------------------------- tiny in‑process GUI helpers ----------------------------
 class ClickAnimation(QWidget):
-    def __init__(self, pos: QPoint, life_ms: int, size_px: int = 50):
+    def __init__(self, pos: QPoint, life_ms: int):
         super().__init__(None,
             Qt.FramelessWindowHint | Qt.Tool | Qt.WindowStaysOnTopHint
             | Qt.WindowTransparentForInput)
         self.setAttribute(Qt.WA_TranslucentBackground)
+        
+        if not CLICK_GIF.exists():
+            print(f"Error: click.gif not found at {CLICK_GIF}")
+            return
+            
+        try:
+            # 创建标签显示GIF
+            self.label = QLabel(self)
+            self.movie = QMovie(str(CLICK_GIF))
+            
+            # 获取原始尺寸并打印（仅供参考）
+            self.movie.jumpToFrame(0)
+            original_size = self.movie.currentPixmap().size()
+            print(f"GIF original size: {original_size.width()}x{original_size.height()}")
+            
+            # 将GIF缩放到30x30像素
+            target_size = QSize(50, 50)
+            self.movie.setScaledSize(target_size)
+            
+            # 设置标签尺寸和GIF
+            self.label.setMovie(self.movie)
+            self.label.setFixedSize(target_size)
+            
+            # 设置窗口大小和位置
+            self.resize(target_size)
+            self.move(pos.x() - 15, pos.y() - 15)  # 居中显示
+            
+            # 提高播放性能
+            self.movie.setCacheMode(QMovie.CacheAll)
+            
+            # 开始播放动画
+            self.movie.start()
+            
+            # 设置定时器关闭窗口
+            QTimer.singleShot(life_ms, self.close)
+            
+            self.show()
+            self.raise_()
+            print(f"Click animation created at ({pos.x()}, {pos.y()}), size: 30x30, duration: {life_ms}ms")
+        except Exception as e:
+            print(f"Error creating click animation: {str(e)}")
 
-        self.label  = QLabel(self)
-        movie       = QMovie(str(CLICK_GIF))
-        movie.setScaledSize(QSize(size_px, size_px))
-        self.label.setMovie(movie)
-        self.label.setFixedSize(size_px, size_px)
+# ---------- 外部接口 ----------
+_app = None
+def _ensure_app():
+    global _app
+    if _app is None:
+        if QApplication.instance() is None:
+            print("Creating new QApplication instance")
+            _app = QApplication(sys.argv)
+        else:
+            print("Using existing QApplication instance")
+            _app = QApplication.instance()
 
-        self.resize(size_px, size_px)
-        self.move(pos.x() - size_px//2, pos.y() - size_px//2)
+# Keep references to animations to prevent garbage collection
+_active_animations = []
 
-        movie.setCacheMode(QMovie.CacheAll)
-        movie.start()
-        QTimer.singleShot(life_ms, self.close)
-        self.show()
-        self.raise_()
+def show_click(x: int, y: int, duration_ms: int = 2000, existing_ms: int = 2000):  # 增加默认播放时间和静止时间
+    """非阻塞式点击动画：立即返回，动画在后台运行
+    
+    Args:
+        x, y          : 屏幕坐标
+        duration_ms   : 动画播放时长
+        existing_ms   : 动画结束后静止显示的时间
+    """
+    print(f"Attempting to show click at ({x}, {y})")
+    
+    if not CLICK_GIF.exists():
+        raise FileNotFoundError(f"click.gif not found at {CLICK_GIF}")
+        
+    _ensure_app()
+    
+    try:
+        # 总生存时间 = 动画时间 + 静止显示时间
+        total_life_ms = duration_ms + existing_ms
+        animation = ClickAnimation(QPoint(x, y), total_life_ms)
+        
+        # Store reference to prevent garbage collection
+        global _active_animations
+        _active_animations.append(animation)
+        
+        # Set up cleanup after animation completes + existing time
+        QTimer.singleShot(total_life_ms + 150, lambda: _clean_animation(animation))
+        
+        print(f"Click animation started (non-blocking, will exist for {total_life_ms}ms)")
+    except Exception as e:
+        print(f"Error during show_click: {str(e)}")
 
-# ------------------------- worker functions that live in a **child** -----------------
-def _worker_click(x, y, duration_ms, existing_ms):
-    app = QApplication(sys.argv)
-    total = duration_ms + existing_ms
-    widget = ClickAnimation(QPoint(x, y), total)  # Store in variable to prevent garbage collection
-    QTimer.singleShot(total + 200, app.quit)      # close event‑loop afterwards
-    app.exec()
 
-def _worker_move(x1, y1, x2, y2, duration_ms, existing_ms):
-    app     = QApplication(sys.argv)
-    total   = duration_ms + existing_ms
-    widget  = ClickAnimation(QPoint(x1, y1), total)
+def _clean_animation(animation):
+    """Remove animation from reference list after it completes"""
+    global _active_animations
+    if animation in _active_animations:
+        _active_animations.remove(animation)
+    print("Animation cleaned up")
 
-    anim = QPropertyAnimation(widget, b"pos")
-    anim.setDuration(duration_ms)
-    anim.setStartValue(widget.pos())
-    anim.setEndValue(QPoint(x2 - widget.width()//2, y2 - widget.height()//2))
-    anim.setEasingCurve(QEasingCurve.OutQuad)
-    anim.start()
 
-    QTimer.singleShot(total + 200, app.quit)
-    app.exec()
+# ---------- 新增函数 ----------
+def show_move_to(x1: int, y1: int, x2: int, y2: int, duration_ms: int = 1000, existing_ms: int = 3000):
+    """
+    非阻塞式移动动画：在 (x1, y1) 处出现光标 GIF，
+    并在 duration_ms 毫秒内平滑移动到 (x2, y2)，
+    然后在终点静止显示 existing_ms 毫秒。
+    立即返回，动画在后台运行。
 
-# ------------------------------- public API (non‑blocking) ---------------------------
-def show_click(x: int, y: int, duration_ms: int = 800, existing_ms: int = 800):
-    if not CLICK_GIF.exists():
-        raise FileNotFoundError(f"GIF not found at {CLICK_GIF}")
-    mp.get_context("spawn").Process(
-        target=_worker_click,
-        args=(x, y, duration_ms, existing_ms),
-        daemon=False                   # keep running even if parent exits
-    ).start()
+    Args:
+        x1, y1        : 起点屏幕坐标
+        x2, y2        : 终点屏幕坐标
+        duration_ms   : 移动总时长
+        existing_ms   : 移动结束后在终点静止显示的时间
+    """
+    print(f"Attempting to move click from ({x1}, {y1}) → ({x2}, {y2}) "
+          f"in {duration_ms} ms, then stay for {existing_ms} ms")
 
-def show_move_to(x1: int, y1: int, x2: int, y2: int,
-                 duration_ms: int = 1000, existing_ms: int = 800):
     if not CLICK_GIF.exists():
-        raise FileNotFoundError(f"GIF not found at {CLICK_GIF}")
-    mp.get_context("spawn").Process(
-        target=_worker_move,
-        args=(x1, y1, x2, y2, duration_ms, existing_ms),
-        daemon=False
-    ).start()
+        raise FileNotFoundError(f"click.gif not found at {CLICK_GIF}")
 
+    _ensure_app()
+
+    # 总生存时间 = 动画时间 + 静止显示时间
+    total_life_ms = duration_ms + existing_ms
+    widget = ClickAnimation(QPoint(x1, y1), total_life_ms)
+    
+    # 用 QPropertyAnimation 平滑移动窗口
+    anim = QPropertyAnimation(widget, b"pos")
+    anim.setDuration(duration_ms)
+    # ClickAnimation 内部已经向左上偏移了 15px，这里沿用同样的偏移
+    anim.setStartValue(QPoint(x1 - 15, y1 - 15))
+    anim.setEndValue(QPoint(x2 - 15, y2 - 15))
+    anim.setEasingCurve(QEasingCurve.OutQuad)     # 可自行更换缓动曲线
+    
+    # Store references to both widget and animation to prevent garbage collection
+    global _active_animations
+    # Store them as a tuple to keep both references
+    animation_pair = (widget, anim)
+    _active_animations.append(animation_pair)
+    
+    # Clean up both widget and animation after completion of total life time
+    def cleanup():
+        if animation_pair in _active_animations:
+            _active_animations.remove(animation_pair)
+        print("Move animation cleaned up")
+    
+    # Connect finished signal only to print a message
+    anim.finished.connect(lambda: print("Movement finished, now staying still"))
+    
+    # Start the animation
+    anim.start()
+    
+    # Process events immediately to kickstart the animation
+    QApplication.processEvents()
+    
+    # Set up final cleanup after animation + existing time
+    QTimer.singleShot(total_life_ms, cleanup)
+    
+    print("Move-to animation started (non-blocking)")
 
 
+# ---------- 命令行测试 ----------
 if __name__ == "__main__":
-    # from click_anim_async import show_click
-    show_click(500, 500)
-    show_move_to(300, 300, 600, 600)
+    # 确保应用程序实例存在
+    _ensure_app()
+    
+    # 测试点击
+    print("Testing non-blocking click animation...")
+    x, y = 500, 500
+    show_click(x, y)
+    
+    # 测试同时运行两个动画
+    print("\nTesting simultaneous animations...")
+    x1, y1 = 200, 200
+    x2, y2 = 600, 600
+    # show_click(x1, y1)
+    show_move_to(x1, y1, x2, y2, duration_ms=2000)
+    
+    # # 测试先移动，然后点击
+    print("\nTesting sequence with pyautogui simulation...")
+    x3, y3 = 800, 300
+    x4, y4 = 400, 500
+    
+    # 启动移动动画
+    show_move_to(x3, y3, x4, y4, duration_ms=1500)
+    
+    # 模拟移动完成后的点击动画（延迟1.5秒）
+    QTimer.singleShot(1500, lambda: show_click(x4, y4))
+    
+    # 保持主程序运行，等待所有动画完成
+    print("\nWaiting for all animations to complete...")
+    loop = QEventLoop()
+    # 等待足够长的时间，确保所有动画都完成（最长的动画是2000ms + 清理时间）
+    QTimer.singleShot(4000, loop.quit)
+    loop.exec()
+    
+    print("All animations completed, exiting test.")

computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py

@@ -249,6 +249,9 @@ class TeachmodeExecutor:
             else:
                 parsed_action_list.append(action)
 
+        # parsed_action_list.extend([{"action": "key_down_windll", "text": "alt", "coordinate": None}])
+        # parsed_action_list.extend(action_list)
+        # parsed_action_list.extend([{"action": "key_up_windll", "text": "alt", "coordinate": None}])
         return parsed_action_list
 
 

computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/test_capture.py

@@ -1,8 +1,8 @@
-from PIL import ImageGrab
-
-bbox=(2560, 366, 2560+1920, 366+1080)
-        
-screenshot = ImageGrab.grab(bbox=bbox, all_screens=True)
-
-screenshot = screenshot.convert('RGB')
+from PIL import ImageGrab
+
+bbox=(2560, 366, 2560+1920, 366+1080)
+        
+screenshot = ImageGrab.grab(bbox=bbox, all_screens=True)
+
+screenshot = screenshot.convert('RGB')
 screenshot.save("screenshot.png")

computer_use_ootb_internal/computer_use_demo/tools/computer.py

@@ -317,6 +317,7 @@ class ComputerTool(BaseAnthropicTool):
                 pyautogui.mouseDown()
                 time.sleep(1)
                 pyautogui.mouseUp()
+                show_click(x, y)
             elif action == "scroll_down":
                 pyautogui.scroll(-200)  # Adjust scroll amount as needed
                 return ToolResult(output="Scrolled down")

computer_use_ootb_internal/example_websocket_js.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Teachmode Client</title>
+    <!-- include socket.io client -->
+    <script src="https://cdn.socket.io/4.5.4/socket.io.min.js"></script>
+  </head>
+  <body>
+    <h1>Teachmode WebSocket Test</h1>
+    <div id="messages"></div>
+
+    <script>
+      const socket = io("http://localhost:5000"); // Your server’s URL/port
+
+      // Listen for partial responses
+      socket.on("partial_response", (data) => {
+        console.log("Got partial:", data);
+        const div = document.createElement("div");
+        div.innerText = "Assistant: " + data.content;
+        document.getElementById("messages").appendChild(div);
+      });
+
+      // Listen for done event
+      socket.on("done", (data) => {
+        console.log("Task completed:", data);
+        const div = document.createElement("div");
+        div.innerText = "TASK DONE!";
+        document.getElementById("messages").appendChild(div);
+      });
+
+      // When connected, emit run_teachmode
+      socket.on("connect", () => {
+        console.log("Connected to server. Emitting run_teachmode event...");
+        socket.emit("run_teachmode", {
+          user_input: "Hello, I'd like to do X, Y, Z."
+          // you can include model, task, user_id, trace_id, etc. if desired
+        });
+      });
+    </script>
+  </body>
+</html>

computer_use_ootb_internal/requirements-lite.txt

@@ -15,5 +15,4 @@ uiautomation
 pywinauto
 textdistance
 matplotlib
-litellm
-PySide6
+litellm

computer_use_ootb_internal/run_teachmode_ootb_args.py

@@ -41,7 +41,7 @@ def simple_teachmode_sampling_loop(
     if "star_rail" in user_id or "star_rail" in user_id:
         full_screen_game_mode = 1
     
-    if "star_rail_dev" in trace_id or "star_rail_dev" in user_id or "hero_case" in user_id:
+    if "star_rail_dev" in trace_id or "star_rail_dev" in user_id or "hero_case" in trace_id:
         full_screen_game_mode = 2
 
     print(f"Full Screen Game Mode: {full_screen_game_mode}")
@@ -102,7 +102,7 @@ def simple_teachmode_sampling_loop(
 
         try:
             step_plan = infer_server_response["generated_plan"]
-            step_info = infer_server_response["generated_plan"]["step_info"]
+            step_info = infer_server_response["generated_action"]["step_info"]
             step_action = infer_server_response["generated_action"]["content"]
             step_traj_idx = infer_server_response["current_traj_step"]
 
@@ -125,6 +125,8 @@ def simple_teachmode_sampling_loop(
 
 
 
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Run a synchronous sampling loop for assistant/tool interactions in teach-mode."
@@ -136,6 +138,8 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "--task",
+        # default="Help me to complete the extraction of the viewer data of Downald Trump's first video on youtube,\
+        # fill in the excel sheet.",
         default="Click on the Google Chorme icon",
         help="The task to be completed by the assistant (e.g., 'Complete some data extraction.').",
     )

computer_use_ootb_internal/service_teachmode.py

@@ -0,0 +1,194 @@
+import argparse
+import time
+import json
+import threading
+
+from flask import Flask, request, jsonify
+from flask_socketio import SocketIO, emit
+
+from screeninfo import get_monitors
+from computer_use_ootb_internal.computer_use_demo.tools.computer import get_screen_details
+from computer_use_ootb_internal.run_teachmode_ootb_args import simple_teachmode_sampling_loop
+
+###############################################################################
+#                                Shared State
+###############################################################################
+class SharedState:
+    def __init__(self):
+        self.args = None  # Will hold argparse-like namespace
+        self.messages = []  # If you want to store a global chat or last session
+
+shared_state = SharedState()
+
+###############################################################################
+#                       Flask + SocketIO Application Setup
+###############################################################################
+app = Flask(__name__)
+app.config["SECRET_KEY"] = "some-secret-key"  # In production, change this
+socketio = SocketIO(app, cors_allowed_origins="*")
+
+###############################################################################
+#                            Utility Functions
+###############################################################################
+def setup_default_args():
+    """
+    Creates argparse-like defaults. 
+    You can also parse real CLI args if you wish.
+    """
+    parser = argparse.ArgumentParser(description="Teachmode SocketIO Server.")
+    parser.add_argument("--model", default="teach-mode-gpt-4o")
+    parser.add_argument("--task", default="Help me complete data extraction on YouTube video.")
+    parser.add_argument("--selected_screen", type=int, default=0)
+    parser.add_argument("--user_id", default="liziqi")
+    parser.add_argument("--trace_id", default="default_trace")
+    parser.add_argument("--api_key_file", default="api_key.json")
+    parser.add_argument("--api_keys", default="")
+    parser.add_argument(
+        "--server_url",
+        default="http://ec2-44-234-43-86.us-west-2.compute.amazonaws.com/generate_action",
+        help="Server URL for the session (local='http://localhost:5000/generate_action', \
+              aws='http://ec2-44-234-43-86.us-west-2.compute.amazonaws.com/generate_action').",
+    )
+
+    # If you really want to parse sys.argv, do parser.parse_args().
+    # But you can also return the defaults for this example:
+    return parser.parse_args([])
+
+def apply_args_to_state(args):
+    """
+    Helper that prints or logs relevant arguments and stores them in shared_state.
+    """
+    print("[apply_args_to_state] Applying arguments:", args)
+    shared_state.args = args
+
+def run_teachmode_task(user_input):
+    """
+    Calls simple_teachmode_sampling_loop and emits partial responses over SocketIO.
+    """
+    # 1) Log or store user input
+    print(f"[run_teachmode_task] Received user_input: {user_input}")
+    # Optionally store or reset message history for this session
+    shared_state.messages = [{"role": "user", "content": user_input}]
+
+    # 2) Grab arguments from shared_state
+    args = shared_state.args
+    if not args:
+        print("[run_teachmode_task] No arguments in shared_state, applying defaults.")
+        args = setup_default_args()
+        apply_args_to_state(args)
+
+    # 3) Run the sampling loop
+    print(f"[run_teachmode_task] Starting the sampling loop with task: {args.task}")
+    sampling_loop = simple_teachmode_sampling_loop(
+        model=args.model,
+        task=args.task,
+        selected_screen=args.selected_screen,
+        user_id=args.user_id,
+        trace_id=args.trace_id,
+        api_keys=args.api_keys,
+        server_url=args.server_url
+    )
+
+    # 4) Send partial responses
+    for loop_msg in sampling_loop:
+        print(f"[run_teachmode_task] Emitting partial response: {loop_msg}")
+        # You can store it in shared_state messages
+        shared_state.messages.append({"role": "assistant", "content": loop_msg})
+        # Emit immediately so the client sees partial responses
+        emit("partial_response", {"role": "assistant", "content": loop_msg})
+        time.sleep(1)  # Optional delay to simulate real-time streaming
+
+    # 5) Done event
+    print("[run_teachmode_task] Completed all messages.")
+    emit("done", {"messages": shared_state.messages, "status": "completed"})
+
+###############################################################################
+#                           HTTP Endpoint: update_params
+###############################################################################
+@app.route("/update_params", methods=["POST"])
+def update_parameters():
+    """
+    HTTP endpoint that allows updating the parameters (like Gradio's /update_params).
+    Expects JSON body with fields matching the argparse Namespace (model, task, etc.)
+    """
+    data = request.json
+    if not data:
+        return jsonify({"status": "error", "message": "No JSON provided."}), 400
+
+    # Build an argparse.Namespace from the JSON keys
+    # Fallback to the existing arguments if some keys are missing
+    old_args = shared_state.args or setup_default_args()
+    new_args_dict = {**vars(old_args), **data}  # Merge old with new
+    new_args = argparse.Namespace(**new_args_dict)
+    apply_args_to_state(new_args)
+
+    return jsonify({
+        "status": "success",
+        "message": "Parameters updated",
+        "new_args": vars(new_args)
+    })
+
+###############################################################################
+#                           HTTP Endpoint: get_messages
+###############################################################################
+@app.route("/get_messages", methods=["GET"])
+def get_messages():
+    """
+    Example new function: returns the current chat messages in shared_state.
+    """
+    return jsonify(shared_state.messages)
+
+###############################################################################
+#                           HTTP Endpoint: clear_messages
+###############################################################################
+@app.route("/clear_messages", methods=["POST"])
+def clear_messages():
+    """
+    Example new function: clears the stored chat messages in shared_state.
+    """
+    shared_state.messages = []
+    return jsonify({"status": "success", "message": "Chat history cleared."})
+
+###############################################################################
+#                        SocketIO Event: run_teachmode
+###############################################################################
+@socketio.on("run_teachmode")
+def handle_run_teachmode(data):
+    """
+    Websocket event that starts the teachmode sampling loop.
+    `data` can include e.g. {"user_input": "..."}.
+    """
+    user_input = data.get("user_input", "Hello, let's start!")
+    run_teachmode_task(user_input)
+
+###############################################################################
+#                        SocketIO Event: connect
+###############################################################################
+@socketio.on("connect")
+def on_connect():
+    print("[SocketIO] Client connected.")
+
+@socketio.on("disconnect")
+def on_disconnect():
+    print("[SocketIO] Client disconnected.")
+
+###############################################################################
+#                                   Main
+###############################################################################
+def main():
+    # Pre-populate shared_state with default arguments
+    args = setup_default_args()
+    apply_args_to_state(args)
+
+    # Optional: Preload screen info if needed
+    screens = get_monitors()
+    print("Detected screens:", screens)
+    screen_names, primary_index = get_screen_details()
+    print("Screen names:", screen_names, "Default selected index:", primary_index)
+
+    # Run the Flask-SocketIO app
+    # eventlet is the default async_mode if installed, but we specify it explicitly.
+    socketio.run(app, host="0.0.0.0", port=5001, debug=True)
+
+if __name__ == "__main__":
+    main()

computer_use_ootb_internal/service_teachmode_test.py

@@ -0,0 +1,41 @@
+import socketio
+
+# Create a Socket.IO client instance
+sio = socketio.Client()
+
+@sio.on('connect')
+def on_connect():
+    print("Connected to the server.")
+    # Once connected, send the event to start the teachmode process:
+    data = {
+        "user_input": "Hello, I'd like to open the Chrome browser."
+        # You can add more parameters here if needed, e.g.:
+        # "model": "teach-mode-gpt-4o",
+        # "task": "Some task",
+        # "user_id": "my_user",
+        # etc.
+    }
+    print("Emitting 'run_teachmode' event with data:", data)
+    sio.emit("run_teachmode", data)
+
+@sio.on('partial_response')
+def on_partial_response(data):
+    print("[partial_response] =>", data)
+
+@sio.on('done')
+def on_done(data):
+    print("[done] =>", data)
+    # Since the process is completed, you can disconnect:
+    sio.disconnect()
+
+@sio.on('disconnect')
+def on_disconnect():
+    print("Disconnected from server.")
+
+
+if __name__ == "__main__":
+    # Connect to the Socket.IO server (adapt host/port as needed):
+    sio.connect("http://localhost:5001")
+
+    # Keep the client alive to receive events
+    sio.wait()

{computer_use_ootb_internal-0.0.103.dist-info → computer_use_ootb_internal-0.0.104.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: computer-use-ootb-internal
-Version: 0.0.103
+Version: 0.0.104
 Summary: Computer Use OOTB
 Author-email: Siyuan Hu <siyuan.hu.sg@gmail.com>
 Requires-Python: >=3.11
@@ -14,15 +14,14 @@ Requires-Dist: matplotlib
 Requires-Dist: opencv-python
 Requires-Dist: pre-commit==3.8.0
 Requires-Dist: pyautogui==0.9.54
-Requires-Dist: pyside6
 Requires-Dist: pytest-asyncio==0.23.6
 Requires-Dist: pytest==8.3.3
-Requires-Dist: pywinauto
+Requires-Dist: pywinauto; sys_platform == 'win32'
 Requires-Dist: ruff==0.6.7
 Requires-Dist: screeninfo
 Requires-Dist: streamlit>=1.38.0
 Requires-Dist: textdistance
-Requires-Dist: uiautomation
+Requires-Dist: uiautomation; sys_platform == 'win32'
 Provides-Extra: dev
 Requires-Dist: pytest-asyncio>=0.23.6; extra == 'dev'
 Requires-Dist: pytest>=8.3.3; extra == 'dev'

{computer_use_ootb_internal-0.0.103.dist-info → computer_use_ootb_internal-0.0.104.dist-info}/RECORD

@@ -1,17 +1,19 @@
 computer_use_ootb_internal/README.md,sha256=FxpW95lyub2iX73ZDfK6ML7SdEKg060H5I6Grub7li4,31
-computer_use_ootb_internal/app_teachmode.py,sha256=zmUPvFjqdhysnN1bD2QQhaAKONnAtMFd03Rb9gF1l6c,15646
+computer_use_ootb_internal/app_teachmode.py,sha256=k4Y5ecIPmgQrtBkSV_xPHb0C6tssdEWfqI7uQsW2ku8,20963
 computer_use_ootb_internal/app_teachmode_gradio.py,sha256=zAw-n3s20j1Jr0S4TzXHwllKV6APJ8HEHB1KqBuzriY,7907
 computer_use_ootb_internal/dependency_check.py,sha256=y8RMEP6RXQzTgU1MS_1piBLtz4J-Hfn9RjUZg59dyvo,1333
-computer_use_ootb_internal/requirements-lite.txt,sha256=5DAHomz4A_P2BmTIXNkNqkHbnIF0AyZ4_1XAlb1LaYs,290
-computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=UEiwLSdERzBqbCH2Em20b4UNzAb__S8cJSnkceVcBsY,6686
-computer_use_ootb_internal/computer_use_demo/animation/click_animation.py,sha256=QR_DEDk7bVON5EQ_xsJGrxNa3NoxqubYyXPFRB12pmQ,3183
+computer_use_ootb_internal/example_websocket_js.html,sha256=BLYwDExVlgiAX4vXVXW3RuP5KD8FXE4EFXIl54bwF7w,1322
+computer_use_ootb_internal/requirements-lite.txt,sha256=2C4OH_GRzuDsR-c9VpJee_te-VqLOf-KDk3LXKi3qdk,282
+computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=djr4E7_G_qG9H6qmRz1mrM9Yrcxf1tnlSPH5ZqykF5Y,6845
+computer_use_ootb_internal/service_teachmode.py,sha256=e81zp3B7CA2nMjA-3qoKF4P1hlNGRI7P1tBo1j21FBk,8082
+computer_use_ootb_internal/service_teachmode_test.py,sha256=zpfBFFKD9WGLX4m77ajOBfmczpYsCa3_qTBweeSNRV8,1112
+computer_use_ootb_internal/computer_use_demo/animation/click_animation.py,sha256=j3v-CrFp61mZiJ4-eT06txkOoguZipbakrQFj2d_mbk,7759
 computer_use_ootb_internal/computer_use_demo/animation/icons8-select-cursor-transparent-96.gif,sha256=4LfwsfFQnREXrNRs32aJU2jO65JXianJoL_8q7-8elg,30966
-computer_use_ootb_internal/computer_use_demo/animation/test_animation.py,sha256=SOJz2yffXTkjuAHqk0IZLcMriR0KQYTo7W1b8wGyRGY,1222
-computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=Rr_I0Uk8hsqTmq3Fr3KOtjkXTqSb4pkNxmutENbv8dc,16368
+computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=y7lg_PjMif2WwCKWWC7g8Ys2zPRMh08Vtt42fStujY4,16623
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/__init__.py,sha256=h2CNeuACklxVpJC65QR8_6AvSybEZLmeO45hY_-lLBs,61
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/gui_capture.py,sha256=CxFJbsSb68ERKH7-C4RaaZy7FIhhzrzGx5qQJ4C37cA,13907
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/gui_parser.py,sha256=KSTJ0cMwh3ahUMzHRaDgA2sVNUL4MNlF7qEBGN3G0SI,28993
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/test_capture.py,sha256=26uxu70di9Me-jseym5ejF7RNIbP35uPEiipN5qamIc,211
+computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/test_capture.py,sha256=YbLpuD-mSFiOU7j6HKrtX05oRP9ciEBxxGIsyse-nPI,204
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/uia_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py,sha256=GEA1ES7vOpHBg_Suxpl99reh34kRG4RQpp072JQBK5c,9787
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/icon_detection/icon_detection.py,sha256=ysTgvtjE1XM7QSrLLy1HD0i6_7iOb9GME5FWJljrJg0,9752
@@ -31,12 +33,12 @@ computer_use_ootb_internal/computer_use_demo/tools/base.py,sha256=QDqpuuKlhUKJT2
 computer_use_ootb_internal/computer_use_demo/tools/bash.py,sha256=rHetQ80_v-TTi-1oxIA7ncFEwJxFTh8FJCErIoZbGeY,4236
 computer_use_ootb_internal/computer_use_demo/tools/collection.py,sha256=8RzHLobL44_Jjt8ltXS6I8XJlEAQOfc75dmnDUaHE-8,922
 computer_use_ootb_internal/computer_use_demo/tools/colorful_text.py,sha256=cvlmnhAImDTwoRRwhT5au7mNFhfAD7ZfeoDEVdVzDKw,892
-computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=kpcp3orAdSwzBJMvL8zt_OgF6kT9UdZaJuqkjja7bic,25493
+computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=bKOkCtE4iYmPpRxldslAAa5yQOs6NkhITJYCfZllWXI,25526
 computer_use_ootb_internal/computer_use_demo/tools/computer_marbot.py,sha256=zZuWz9ArfP3Zss-afnscrPkgCtB5UWbCy7HwAOvO2bo,5970
 computer_use_ootb_internal/computer_use_demo/tools/edit.py,sha256=b0PwUitxckHCQqFP3ZwlthWdqNkn7WETeTHeB6-o98c,11486
 computer_use_ootb_internal/computer_use_demo/tools/run.py,sha256=xhXdnBK1di9muaO44CEirL9hpGy3NmKbjfMpyeVmn8Y,1595
 computer_use_ootb_internal/computer_use_demo/tools/screen_capture.py,sha256=L8qfvtUkPPQGt92N-2Zfw5ZTDBzLsDps39uMnX3_uSA,6857
-computer_use_ootb_internal-0.0.103.dist-info/METADATA,sha256=un-dDRkh_CxGGAaFc-BSNKzMVpRvl0YabB3qZEJRbq4,910
-computer_use_ootb_internal-0.0.103.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-computer_use_ootb_internal-0.0.103.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
-computer_use_ootb_internal-0.0.103.dist-info/RECORD,,
+computer_use_ootb_internal-0.0.104.dist-info/METADATA,sha256=xXYZQkg2ZeJ3P8aTG4q5XbA48RjL-ZK9e3z1BUSoLHk,937
+computer_use_ootb_internal-0.0.104.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+computer_use_ootb_internal-0.0.104.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
+computer_use_ootb_internal-0.0.104.dist-info/RECORD,,

computer_use_ootb_internal/computer_use_demo/animation/test_animation.py

@@ -1,40 +0,0 @@
-"""
-Test script to verify cursor animation is working
-"""
-import asyncio
-import sys
-import time
-from pathlib import Path
-from computer_use_ootb_internal.computer_use_demo.tools.computer import ComputerTool
-
-async def test_animations():
-    
-    # Initialize the computer tool
-    computer = ComputerTool()
-    
-    # Test mouse move animation
-    print("Testing mouse move animation...")
-    await computer(action="mouse_move_windll", coordinate=(500, 500))
-    print("Waiting 2 seconds...")
-    await asyncio.sleep(2)
-    
-    # Test click animation
-    print("Testing click animation...")
-    await computer(action="left_click_windll", coordinate=(700, 300))
-    print("Waiting 2 seconds...")
-    await asyncio.sleep(2)
-    
-    # Test another move
-    print("Testing move and click sequence...")
-    await computer(action="mouse_move_windll", coordinate=(300, 300))
-    await asyncio.sleep(1)
-    await computer(action="left_click_windll", coordinate=(300, 300))
-    
-    # Wait for animations to complete
-    print("Waiting for animations to complete...")
-    await asyncio.sleep(3)
-    
-    print("Test completed")
-
-if __name__ == "__main__":
-    asyncio.run(test_animations())