computer-use-ootb-internal 0.0.108__py3-none-any.whl → 0.0.109.post1__py3-none-any.whl

computer_use_ootb_internal/run_teachmode_ootb_args.py

@@ -2,16 +2,13 @@ import argparse
 import time
 import json
 import platform
-from typing import Callable
-from collections.abc import Callable
 import uuid
 import datetime
 from datetime import datetime, timedelta, timezone
 
 from computer_use_ootb_internal.computer_use_demo.executor.teachmode_executor import TeachmodeExecutor
-from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.gui_capture import get_screenshot
-from computer_use_ootb_internal.computer_use_demo.gui_agent.llm_utils.oai import encode_image
-from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.icon_detection.icon_detection import get_screen_resize_factor
+from computer_use_ootb_internal.computer_use_demo.gui_agent.llm_utils.llm_utils import is_image_path
+from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.simple_parser.utils import get_screen_resize_factor
 from computer_use_ootb_internal.computer_use_demo.tools.aws_request import send_request_to_server
 from computer_use_ootb_internal.computer_use_demo.gui_agent.gui_parser.uia_tools.screenshot_service import get_screenshot_external_cmd
 utc_plus_8 = timezone(timedelta(hours=8))
@@ -41,7 +38,7 @@ def simple_teachmode_sampling_loop(
     if "star_rail" in user_id or "star_rail" in user_id:
         full_screen_game_mode = 1
     
-    if "star_rail_dev" in trace_id or "star_rail_dev" in user_id or "hero_case" in user_id:
+    if "star_rail_dev" in trace_id or "star_rail_dev" in user_id or "hero_case" in user_id or "offical" in user_id:
         full_screen_game_mode = 2
 
     print(f"Full Screen Game Mode: {full_screen_game_mode}")
@@ -53,7 +50,7 @@ def simple_teachmode_sampling_loop(
     timestamp = datetime.now(utc_plus_8).strftime("%m%d-%H%M%S")
 
     step_count = 1
-    unique_task_id = f"{timestamp}_uid_{user_id}_tid_{trace_id}_{str(uuid.uuid4())[:4]}"
+    unique_task_id = f"{timestamp}_uid_{user_id}_tid_{trace_id}_{str(uuid.uuid4())[:6]}"
 
     print("[simple_teachmode_sampling_loop] starting task: ", task)
     print(f"[simple_teachmode_sampling_loop] unique_task_id: {unique_task_id}")
@@ -68,6 +65,10 @@ def simple_teachmode_sampling_loop(
 
         uia_meta, sc_path = get_screenshot_external_cmd(selected_screen=selected_screen,
                                                         capture_uia_data=full_screen_game_mode==0)
+        
+        if is_image_path(sc_path):
+            screenshot_message = {"role": "user", "content": sc_path, "type": "image"} 
+            yield screenshot_message
 
         payload = {
             "task_id": unique_task_id,
@@ -102,25 +103,35 @@ def simple_teachmode_sampling_loop(
 
         try:
             step_plan = infer_server_response["generated_plan"]
-            step_info = infer_server_response["generated_plan"]["step_info"]
+            step_reasoning = step_plan["reasoning"]
+            step_info = step_plan["step_info"]
             step_action = infer_server_response["generated_action"]["content"]
             step_traj_idx = infer_server_response["current_traj_step"]
 
         except Exception as e:
             print("Error parsing generated_action content:", e)
             continue
+        
+        plan_message = {"role": "assistant", "content": step_reasoning, "type": "text"}
+        yield plan_message
 
         if step_action.get("action") == "STOP":
             final_sc, final_sc_path = get_screenshot_external_cmd(selected_screen=selected_screen)
-            action_history = []  # reset action history
-            break
 
-        # action_history.append(f"Executing Step: {step_count} - Trajectory Step: {step_traj_idx} - Plan: {step_plan} - Action: {step_action};\n")
+            final_message = {"role": "assistant", "content": "Task completed. Final screenshot:", "type": "text"}
+            yield final_message
+
+            final_sc_message = {"role": "user", "content": final_sc_path, "type": "image"}
+            yield final_sc_message
+
+            # reset action history
+            action_history = []  
+            break
 
         action_history.append(f"Executing guidance trajectory step [{step_traj_idx}], Plan: {step_info}, Action: {step_action};\n")
 
-        for message in executor({"role": "assistant", "content": step_action}):
-            yield message
+        for exec_message in executor({"role": "assistant", "content": step_action}):
+            yield exec_message
 
         step_count += 1
     

{computer_use_ootb_internal-0.0.108.dist-info → computer_use_ootb_internal-0.0.109.post1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: computer-use-ootb-internal
-Version: 0.0.108
+Version: 0.0.109.post1
 Summary: Computer Use OOTB
 Author-email: Siyuan Hu <siyuan.hu.sg@gmail.com>
 Requires-Python: >=3.11

{computer_use_ootb_internal-0.0.108.dist-info → computer_use_ootb_internal-0.0.109.post1.dist-info}/RECORD

@@ -1,21 +1,16 @@
 computer_use_ootb_internal/README.md,sha256=FxpW95lyub2iX73ZDfK6ML7SdEKg060H5I6Grub7li4,31
-computer_use_ootb_internal/app_teachmode.py,sha256=CyNUk9AqbmBtOvYsEHG16wBH6F-Vje3CiMPnJ9oB-b8,19549
-computer_use_ootb_internal/app_teachmode_gradio.py,sha256=zAw-n3s20j1Jr0S4TzXHwllKV6APJ8HEHB1KqBuzriY,7907
+computer_use_ootb_internal/app_teachmode.py,sha256=Yb5LtyaW4agGsfTmVvnIAkERXILkg3KkUZf7yI1dW-g,19253
+computer_use_ootb_internal/app_teachmode_gradio.py,sha256=cmFpBrkdlZxOQADWveVdIaaNqaBD8IVs-xNLJogU7F8,7909
 computer_use_ootb_internal/dependency_check.py,sha256=y8RMEP6RXQzTgU1MS_1piBLtz4J-Hfn9RjUZg59dyvo,1333
 computer_use_ootb_internal/requirements-lite.txt,sha256=5DAHomz4A_P2BmTIXNkNqkHbnIF0AyZ4_1XAlb1LaYs,290
-computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=BUGc74mVUt9NffbCiNVe9pDCcwigPQ3fqWU6rO9DT0w,6798
-computer_use_ootb_internal/computer_use_demo/animation/click_animation.py,sha256=99Ehm7HKXCYqLhE3KgdyRcilF3K4dMycKZ14l-CB3_I,3235
+computer_use_ootb_internal/run_teachmode_ootb_args.py,sha256=7Dj0iY4GG7P03tRKYJ2x9Yvt-PE-b7uyjCAed3SaF3Y,7086
+computer_use_ootb_internal/computer_use_demo/animation/click_animation.py,sha256=tP1gsayFy-CKk10UlrE9RlexwlHWiHQUe5Ogg4vQvSg,3234
 computer_use_ootb_internal/computer_use_demo/animation/icons8-select-cursor-transparent-96.gif,sha256=4LfwsfFQnREXrNRs32aJU2jO65JXianJoL_8q7-8elg,30966
 computer_use_ootb_internal/computer_use_demo/animation/test_animation.py,sha256=SOJz2yffXTkjuAHqk0IZLcMriR0KQYTo7W1b8wGyRGY,1222
-computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=3IwTm99UUzQ7NLpt4r2g2iiluXWzdn8n4hCwy2XWP78,16426
+computer_use_ootb_internal/computer_use_demo/executor/teachmode_executor.py,sha256=1xver_8cRCrFk85OYqEQ7LUPYs5v2Mvtg944wiGN05A,16562
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/__init__.py,sha256=h2CNeuACklxVpJC65QR8_6AvSybEZLmeO45hY_-lLBs,61
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/gui_capture.py,sha256=CxFJbsSb68ERKH7-C4RaaZy7FIhhzrzGx5qQJ4C37cA,13907
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/gui_parser.py,sha256=KSTJ0cMwh3ahUMzHRaDgA2sVNUL4MNlF7qEBGN3G0SI,28993
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/test_capture.py,sha256=26uxu70di9Me-jseym5ejF7RNIbP35uPEiipN5qamIc,211
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/uia_parser.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py,sha256=GEA1ES7vOpHBg_Suxpl99reh34kRG4RQpp072JQBK5c,9787
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/icon_detection/icon_detection.py,sha256=ysTgvtjE1XM7QSrLLy1HD0i6_7iOb9GME5FWJljrJg0,9752
-computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/panel_recognition/llm_panel_recognize.py,sha256=R1v3I_U9evoS2H8csBnobl4BQdvcwmg4kgElXu5FARg,6054
+computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/simple_parser/utils.py,sha256=OOVxy4Qlbk5q-X9kXFXqt6AmuOMl6FWWqtH269DvJJA,10005
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/screenshot_cli.py,sha256=hrUBH3aeUe9t4EDsMhaUZLPpNLKeEaG4ZvikrFy-bOU,1313
 computer_use_ootb_internal/computer_use_demo/gui_agent/gui_parser/uia_tools/screenshot_service.py,sha256=fJ8G4_-ZLGsHLdLgHCNgjVM1osbfz21L2w8QrcvnuhA,2738
@@ -27,16 +22,16 @@ computer_use_ootb_internal/computer_use_demo/gui_agent/vlm_utils/__init__.py,sha
 computer_use_ootb_internal/computer_use_demo/gui_agent/vlm_utils/run_vlm.py,sha256=9IaX15dhFFOALr0vcsI-EtEU4pDLfdzlmwCy3HRbISA,1441
 computer_use_ootb_internal/computer_use_demo/tools/__init__.py,sha256=Pj8_5L4_PPQK298X4NV3KMbP-84t-bM0pbjEeb5_SJQ,343
 computer_use_ootb_internal/computer_use_demo/tools/aws_request.py,sha256=12UVzeA2PmpZhpy2Pt5Vh48-_q1e8ZjVQux6r3pbAcw,2629
-computer_use_ootb_internal/computer_use_demo/tools/base.py,sha256=QDqpuuKlhUKJT2hPwgEeBAxGTKrlHs1cHYC-QaVoDNo,2013
+computer_use_ootb_internal/computer_use_demo/tools/base.py,sha256=4HHSxDg6GsCoraamWor75ZnoR7tAcRarMv4MzvqMWGU,2099
 computer_use_ootb_internal/computer_use_demo/tools/bash.py,sha256=rHetQ80_v-TTi-1oxIA7ncFEwJxFTh8FJCErIoZbGeY,4236
 computer_use_ootb_internal/computer_use_demo/tools/collection.py,sha256=8RzHLobL44_Jjt8ltXS6I8XJlEAQOfc75dmnDUaHE-8,922
 computer_use_ootb_internal/computer_use_demo/tools/colorful_text.py,sha256=cvlmnhAImDTwoRRwhT5au7mNFhfAD7ZfeoDEVdVzDKw,892
-computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=nGHrwF60nExw6pVhs_gThr5pSObVv_yE-RYPbkGPG6U,25375
+computer_use_ootb_internal/computer_use_demo/tools/computer.py,sha256=Z-kt6XxGIeZJaeaj02ktAcUQL8vG7co2bedzC5pSAsc,26794
 computer_use_ootb_internal/computer_use_demo/tools/computer_marbot.py,sha256=zZuWz9ArfP3Zss-afnscrPkgCtB5UWbCy7HwAOvO2bo,5970
 computer_use_ootb_internal/computer_use_demo/tools/edit.py,sha256=b0PwUitxckHCQqFP3ZwlthWdqNkn7WETeTHeB6-o98c,11486
 computer_use_ootb_internal/computer_use_demo/tools/run.py,sha256=xhXdnBK1di9muaO44CEirL9hpGy3NmKbjfMpyeVmn8Y,1595
 computer_use_ootb_internal/computer_use_demo/tools/screen_capture.py,sha256=L8qfvtUkPPQGt92N-2Zfw5ZTDBzLsDps39uMnX3_uSA,6857
-computer_use_ootb_internal-0.0.108.dist-info/METADATA,sha256=nvsxZ-GpD2eq2iGByq_h8sOss3aiS039NeAIkJeXQxA,910
-computer_use_ootb_internal-0.0.108.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-computer_use_ootb_internal-0.0.108.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
-computer_use_ootb_internal-0.0.108.dist-info/RECORD,,
+computer_use_ootb_internal-0.0.109.post1.dist-info/METADATA,sha256=sK_kPqyLBjBFbinqhhx7t83mYHAHyeyPnQlr-h3PAQA,916
+computer_use_ootb_internal-0.0.109.post1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+computer_use_ootb_internal-0.0.109.post1.dist-info/entry_points.txt,sha256=-AbmawU7IRQuDZihgVMVDrFoY4E6rnXYOUB-5vSeBKs,93
+computer_use_ootb_internal-0.0.109.post1.dist-info/RECORD,,