computer-use-ootb-internal 0.0.169__tar.gz → 0.0.171__tar.gz

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: computer-use-ootb-internal
-Version: 0.0.169
+Version: 0.0.171
 Summary: Computer Use OOTB
 Author-email: Siyuan Hu <siyuan.hu.sg@gmail.com>
 Requires-Python: >=3.11

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "computer-use-ootb-internal"
-version = "0.0.169"
+version = "0.0.171"
 description = "Computer Use OOTB"
 authors = [{ name = "Siyuan Hu", email = "siyuan.hu.sg@gmail.com" }]
 requires-python = ">=3.11"

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/src/computer_use_ootb_internal/app_teachmode.py

@@ -19,7 +19,10 @@ from fastapi.responses import JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from computer_use_ootb_internal.computer_use_demo.tools.computer import get_screen_details
 from computer_use_ootb_internal.run_teachmode_ootb_args import simple_teachmode_sampling_loop
+from computer_use_ootb_internal.computer_use_demo.executor.teachmode_executor import TeachmodeExecutor
 import uvicorn # Assuming uvicorn is used to run FastAPI
+import concurrent.futures
+import asyncio
 
 # --- App Logging Setup ---
 try:
@@ -395,6 +398,62 @@ async def get_status(request: Request):
         status_code=200
     )
 
+@app.post("/exec_computer_tool")
+async def exec_computer_tool(request: Request):
+    logging.info("Received request to /exec_computer_tool")
+    try:
+        data = await request.json()
+        
+        # Extract parameters from the request
+        selected_screen = data.get('selected_screen', 0)
+        full_screen_game_mode = data.get('full_screen_game_mode', 0)
+        response = data.get('response', {})
+        
+        logging.info(f"Executing TeachmodeExecutor with: screen={selected_screen}, mode={full_screen_game_mode}, response={response}")
+        
+        # Create TeachmodeExecutor in a separate process to avoid event loop conflicts
+        # Since TeachmodeExecutor uses asyncio.run() internally, we need to run it in a way
+        # that doesn't conflict with FastAPI's event loop
+        
+        def run_executor():
+            executor = TeachmodeExecutor(
+                selected_screen=selected_screen,
+                full_screen_game_mode=full_screen_game_mode
+            )
+            
+            results = []
+            try:
+                for action_result in executor(response):
+                    results.append(action_result)
+            except Exception as exec_error:
+                logging.error(f"Error executing action: {exec_error}", exc_info=True)
+                return {"error": str(exec_error)}
+            
+            return results
+        
+        # Execute in a thread pool to avoid blocking the event loop
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            results = await asyncio.get_event_loop().run_in_executor(pool, run_executor)
+        
+        if isinstance(results, dict) and "error" in results:
+            return JSONResponse(
+                content={"status": "error", "message": results["error"]},
+                status_code=500
+            )
+            
+        logging.info(f"Action results: {results}")
+        
+        return JSONResponse(
+            content={"status": "success", "results": results},
+            status_code=200
+        )
+    except Exception as e:
+        logging.error("Error processing /exec_computer_tool:", exc_info=True)
+        return JSONResponse(
+            content={"status": "error", "message": f"Internal server error: {str(e)}"},
+            status_code=500
+        )
+
 def process_input():
     global shared_state
     logging.info("process_input thread started.")
@@ -493,9 +552,6 @@ def main():
     parser.add_argument("--trace_id", type=str, default="build_scroll_combat", help="Trace ID for the session")
     parser.add_argument("--api_keys", type=str, default="sk-proj-1234567890", help="API keys")
     parser.add_argument("--server_url", type=str, default="http://ec2-44-234-43-86.us-west-2.compute.amazonaws.com", help="Server URL for the session")
-    # Add arguments for port override and target user
-    parser.add_argument("-p", "--port", type=int, default=None, help="Specify the port to run the server on, overriding user-based calculation.")
-    parser.add_argument("--target_user", type=str, default=None, help="Specify the target username for port calculation if --port is not given.")
 
     args = parser.parse_args()
 
@@ -512,18 +568,14 @@ def main():
     rate_limiter = RateLimiter(interval_seconds=2) # Re-initialize rate limiter
     logging.info(f"Shared state initialized for user: {args.user_id}")
 
-    # --- Port Calculation Logic ---
+    # --- Restore original port calculation logic --- 
     port = 7888 # Default port
     host = "0.0.0.0" # Listen on all interfaces
     
-    if args.port is not None:
-        port = args.port
-        logging.info(f"Using specified port from --port argument: {port}")
-    elif platform.system() == "Windows" and args.target_user is not None:
+    if platform.system() == "Windows":
         try:
-            # Use the --target_user argument for calculation
-            username = args.target_user.lower()
-            logging.info(f"Determining port based on Windows username from --target_user: {username}")
+            username = os.environ["USERNAME"].lower()
+            logging.info(f"Determining port based on Windows username: {username}")
             if username == "altair":
                 port = 14000
             elif username.startswith("guest") and username[5:].isdigit():
@@ -535,10 +587,10 @@ def main():
             else:
                 logging.info(f"Username '{username}' doesn't match specific rules, using default port {port}.")
         except Exception as e:
-             logging.error(f"Error determining port from --target_user '{args.target_user}': {e}. Using default port {port}.", exc_info=True)
+             logging.error(f"Error determining port from username: {e}. Using default port {port}.", exc_info=True)
     else:
-         logging.info(f"--port not specified, and not on Windows or --target_user not specified. Using default port {port}.")
-    # --- End Port Calculation ---
+         logging.info(f"Not running on Windows, using default port {port}.")
+    # --- End of restored port calculation --- 
 
     logging.info(f"Final Host={host}, Port={port}")
 

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/src/computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py

@@ -115,20 +115,21 @@ class TeachmodeExecutor:
                 print(f"Error parsing action output: {e}")
                 return None
     
-    def _parse_actor_output(self, output_text: str | dict) -> Union[List[Dict[str, Any]], None]:
+    def _parse_actor_output(self, action_item: str | dict) -> Union[List[Dict[str, Any]], None]:
         try:
             # refine key: value pairs, mapping to the Anthropic's format
             refined_output = []
             
-            action_item = output_text
-            
+            if type(action_item) == str:
+                action_item = ast.literal_eval(action_item)
+
             print("[_parse_actor_output] Action Item:", action_item)
             
             # sometime showui returns lower case action names
             action_item["action"] = action_item["action"].upper()
             
             if action_item["action"] not in self.supported_action_type:
-                raise ValueError(f"Action {action_item['action']} not supported. Check the output from ShowUI: {output_text}")
+                raise ValueError(f"Action {action_item['action']} not supported. Check the output from Actor: {action_item}")
                 # continue
             
             elif action_item["action"] == "CLICK":  # 1. click -> mouse_move + left_click
@@ -314,44 +315,3 @@ class TeachmodeExecutor:
         return bbox
 
 
-
-# def _make_api_tool_result(
-#     result: ToolResult, tool_use_id: str
-# ) -> BetaToolResultBlockParam:
-#     """Convert an agent ToolResult to an API ToolResultBlockParam."""
-#     tool_result_content: list[BetaTextBlockParam | BetaImageBlockParam] | str = []
-#     is_error = False
-#     if result.error:
-#         is_error = True
-#         tool_result_content = _maybe_prepend_system_tool_result(result, result.error)
-#     else:
-#         if result.output:
-#             tool_result_content.append(
-#                 {
-#                     "type": "text",
-#                     "text": _maybe_prepend_system_tool_result(result, result.output),
-#                 }
-#             )
-#         if result.base64_image:
-#             tool_result_content.append(
-#                 {
-#                     "type": "image",
-#                     "source": {
-#                         "type": "base64",
-#                         "media_type": "image/png",
-#                         "data": result.base64_image,
-#                     },
-#                 }
-#             )
-#     return {
-#         "type": "tool_result",
-#         "content": tool_result_content,
-#         "tool_use_id": tool_use_id,
-#         "is_error": is_error,
-#     }
-
-
-# def _maybe_prepend_system_tool_result(result: ToolResult, result_text: str):
-#     if result.system:
-#         result_text = f"<system>{result.system}</system>\n{result_text}"
-#     return result_text

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/src/computer_use_ootb_internal/computer_use_demo/tools/computer.py

@@ -384,7 +384,7 @@ class ComputerTool(BaseAnthropicTool):
                     show_click(x, y)
                     time.sleep(0.25)
                     self.marbot_auto_gui.click(x=x, y=y)
-                    self.marbot_auto_gui.click(x=x, y=y)
+                    # self.marbot_auto_gui.click(x=x, y=y)
                 return ToolResult(output=f"Left click", action_base_type="click")
 
             elif action == "mouse_move_windll":

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/src/computer_use_ootb_internal/guard_service.py

@@ -688,24 +688,6 @@ class GuardService(win32serviceutil.ServiceFramework):
         task_created_status = "task_unknown"
         immediate_start_status = "start_not_attempted"
         final_status = "failed_unknown"
-        calculated_port = 7888 # Default port
-
-        # --- Calculate Port based on username (mirroring app_teachmode logic) ---
-        try:
-            if user == "altair":
-                calculated_port = 14000
-            elif user.startswith("guest") and user[5:].isdigit():
-                num = int(user[5:])
-                if 1 <= num <= 10:
-                    calculated_port = 14000 + num
-                else:
-                    self.log_warning(f"Internal trigger: User 'guest{num}' out of range (1-10), using default port {calculated_port} for launch.")
-            # else: use default 7888
-            self.log_info(f"Internal trigger: Calculated port {calculated_port} for user '{user}'.")
-        except Exception as port_calc_e:
-            self.log_error(f"Internal trigger: Error calculating port for user '{user}': {port_calc_e}. Using default port {calculated_port}.")
-        # --- End Port Calculation ---
-
 
         try:
             # 1. Ensure scheduled task exists (still useful fallback/persistence)
@@ -760,77 +742,47 @@ class GuardService(win32serviceutil.ServiceFramework):
 
             # 4. Attempt immediate start (User is active and not running)
             immediate_start_status = "start_attempted"
-            self.log_info(f"Internal trigger: User '{user}' is active and not running. Attempting immediate start via PowerShell elevation...")
-            
-            if not self.target_executable_path:
-                self.log_error("Internal trigger: Cannot start process - target executable path not found.")
-                final_status = "failed_exe_not_found"
-                return final_status # Exit early if executable missing
-
+            self.log_info(f"Internal trigger: User '{user}' is active and not running. Attempting immediate start via CreateProcessAsUser...")
             try:
                 token = win32ts.WTSQueryUserToken(session_id)
                 env = win32profile.CreateEnvironmentBlock(token, False)
                 startup = win32process.STARTUPINFO()
-                # Use CREATE_NEW_CONSOLE to make the PS window visible and potentially the UAC prompt
-                creation_flags = win32con.CREATE_NEW_CONSOLE
-                
-                # Command to run PowerShell, which then elevates the target executable
-                target_exe_unquoted = self.target_executable_path.strip('"')
-                target_exe_for_ps = target_exe_unquoted.replace("'", "''") # Escape single quotes for PS
-                target_exe_dir_for_ps = os.path.dirname(target_exe_unquoted).replace("'", "''")
-                
-                # Build the second argument for cmd.exe /K, as used in the successful test command
-                # Format: "<target_exe>" --port <port> --target_user ''<user>'' 
-                cmd_k_second_arg_content = f'""{target_exe_for_ps}"" --port {calculated_port} --target_user ''{user}'''
-                # Escape single quotes within this second arg for the outer PowerShell command string
-                cmd_k_second_arg_for_ps_command = cmd_k_second_arg_content.replace("'", "''")
-
-                # Build the full Start-Process command string matching the working test command
-                start_process_cmd = (
-                    f"Start-Process -FilePath cmd.exe "
-                    f"-ArgumentList @('/K', '{cmd_k_second_arg_for_ps_command}') "
-                    f"-WorkingDirectory '{target_exe_dir_for_ps}' "
-                    f"-Verb RunAs"
-                )
-
-                # Launch powershell.exe directly, tell it to run the Start-Process command, and keep window open
-                # Use outer single quotes for -Command argument value to simplify inner quoting
-                # No need to escape inner double quotes when using outer single quotes
-                # Escape any literal single quotes within start_process_cmd itself (shouldn't be needed here but good practice)
-                start_process_cmd_escaped_for_single_quotes = start_process_cmd.replace("'", "''")
-                lpCommandLine = f'powershell.exe -NoProfile -ExecutionPolicy Bypass -NoExit -Command \'{start_process_cmd_escaped_for_single_quotes}\''
-
-                lpApplicationName = None # Must be None if using lpCommandLine
-                cwd = os.path.dirname(target_exe_unquoted) if os.path.dirname(target_exe_unquoted) else None
+                creation_flags = 0x00000010 # CREATE_NEW_CONSOLE
+                lpApplicationName = None
+                lpCommandLine = f'cmd.exe /K "{self.target_executable_path}"'
+                cwd = os.path.dirname(self.target_executable_path.strip('"')) if os.path.dirname(self.target_executable_path.strip('"')) != '' else None
 
                 # Log details before call
-                self.log_info(f"Internal trigger: Launching PowerShell directly for elevation:")
+                self.log_info(f"Internal trigger: Calling CreateProcessAsUser:")
                 self.log_info(f"  lpCommandLine: {lpCommandLine}")
                 self.log_info(f"  lpCurrentDirectory: {cwd if cwd else 'Default'}")
-                
+
                 hProcess, hThread, dwPid, dwTid = win32process.CreateProcessAsUser(
                     token, lpApplicationName, lpCommandLine, None, None, False,
                     creation_flags, env, cwd, startup
                 )
-                # We get the PID of cmd.exe/powershell.exe here, not the final elevated process.
-                # We can't easily track the elevated process PID across the UAC boundary.
-                self.log_info(f"Internal trigger: CreateProcessAsUser call to initiate elevation succeeded for user '{user}' (Launcher PID: {dwPid}). UAC prompt expected if needed.")
+                self.log_info(f"Internal trigger: CreateProcessAsUser call succeeded for user '{user}' (PID: {dwPid}). Checking existence...")
                 win32api.CloseHandle(hProcess)
                 win32api.CloseHandle(hThread)
 
-                # Assume success if the call didn't raise an exception.
-                # We cannot easily verify if the *elevated* process actually started.
-                immediate_start_status = "start_elevation_initiated"
-                final_status = "success_elevation_initiated" # Report success based on initiating the elevation
+                time.sleep(1)
+                if psutil.pid_exists(dwPid):
+                    self.log_info(f"Internal trigger: Immediate start succeeded for user '{user}' (PID {dwPid}).")
+                    immediate_start_status = "start_success"
+                    final_status = "success" # Overall success
+                else:
+                    self.log_error(f"Internal trigger: Immediate start failed for user '{user}': Process {dwPid} exited immediately.")
+                    immediate_start_status = "start_failed_exited"
+                    final_status = "failed_start_exited"
 
             except Exception as proc_err:
-                 self.log_error(f"Internal trigger: Exception during CreateProcessAsUser for elevation attempt (user '{user}'): {proc_err}", exc_info=True)
+                 self.log_error(f"Internal trigger: Exception during CreateProcessAsUser for user '{user}': {proc_err}", exc_info=True)
                  immediate_start_status = "start_failed_exception"
                  final_status = "failed_start_exception"
             finally:
                  if token: win32api.CloseHandle(token)
             
-            # Combine results (mostly determined by start attempt now)
+            # Combine results (though mostly determined by start attempt now)
             # Example: final_status = f"{task_created_status}_{immediate_start_status}"
             return final_status
 

{computer_use_ootb_internal-0.0.169 → computer_use_ootb_internal-0.0.171}/src/computer_use_ootb_internal/run_teachmode_ootb_args.py

@@ -119,17 +119,21 @@ def simple_teachmode_sampling_loop(
 
         try:
             step_plan = infer_server_response["generated_plan"]
-            step_reasoning = step_plan["reasoning"]
-            step_info = step_plan["step_info"]
+            step_plan_observation = step_plan["observation"]
+            step_plan_reasoning = step_plan["reasoning"]
+            step_plan_info = step_plan["step_info"]
             step_action = infer_server_response["generated_action"]["content"]
             step_traj_idx = infer_server_response["current_traj_step"]
 
+            # chat_visable_content = f"{step_plan_observation}{step_plan_reasoning}"
+
         except Exception as e:
             print("Error parsing generated_action content:", e)
             yield {"role": "assistant", "content": "Error parsing response from Marbot Run server. Exiting.", "type": "error"}
             break
-        
-        yield {"role": "assistant", "content": step_reasoning, "type": "text"}
+
+        yield {"role": "assistant", "content": step_plan_observation, "type": "text"}
+        yield {"role": "assistant", "content": step_plan_reasoning, "type": "text"}
 
         if step_action.get("action") == "STOP":
             final_sc, final_sc_path = get_screenshot_external_cmd(selected_screen=selected_screen)
@@ -141,7 +145,7 @@ def simple_teachmode_sampling_loop(
             action_history = []  
             break
 
-        action_history.append(f"Executing guidance trajectory step [{step_traj_idx}]: {{Plan: {step_info}, Action: {step_action}}}\n")
+        action_history.append(f"Executing guidance trajectory step [{step_traj_idx}]: {{Plan: {step_plan_info}, Action: {step_action}}}\n")
 
         for exec_message in executor({"role": "assistant", "content": step_action}):
             yield exec_message