minitap-mobile-use 2.5.3__py3-none-any.whl → 2.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of minitap-mobile-use might be problematic. Click here for more details.
- minitap/mobile_use/agents/contextor/contextor.py +0 -8
- minitap/mobile_use/agents/cortex/cortex.md +122 -36
- minitap/mobile_use/agents/cortex/cortex.py +32 -17
- minitap/mobile_use/agents/cortex/types.py +18 -4
- minitap/mobile_use/agents/executor/executor.md +3 -3
- minitap/mobile_use/agents/executor/executor.py +10 -3
- minitap/mobile_use/agents/hopper/hopper.md +30 -2
- minitap/mobile_use/agents/hopper/hopper.py +19 -15
- minitap/mobile_use/agents/orchestrator/orchestrator.py +14 -5
- minitap/mobile_use/agents/outputter/outputter.py +13 -3
- minitap/mobile_use/agents/planner/planner.md +20 -9
- minitap/mobile_use/agents/planner/planner.py +12 -5
- minitap/mobile_use/agents/screen_analyzer/human.md +16 -0
- minitap/mobile_use/agents/screen_analyzer/screen_analyzer.py +111 -0
- minitap/mobile_use/clients/ios_client.py +7 -3
- minitap/mobile_use/config.py +87 -24
- minitap/mobile_use/controllers/mobile_command_controller.py +354 -88
- minitap/mobile_use/controllers/platform_specific_commands_controller.py +41 -27
- minitap/mobile_use/controllers/types.py +95 -0
- minitap/mobile_use/graph/graph.py +55 -11
- minitap/mobile_use/graph/state.py +10 -3
- minitap/mobile_use/main.py +12 -4
- minitap/mobile_use/sdk/agent.py +109 -72
- minitap/mobile_use/sdk/examples/smart_notification_assistant.py +59 -10
- minitap/mobile_use/servers/device_hardware_bridge.py +13 -6
- minitap/mobile_use/services/llm.py +5 -2
- minitap/mobile_use/tools/index.py +7 -9
- minitap/mobile_use/tools/mobile/{clear_text.py → focus_and_clear_text.py} +7 -7
- minitap/mobile_use/tools/mobile/{input_text.py → focus_and_input_text.py} +8 -8
- minitap/mobile_use/tools/mobile/long_press_on.py +130 -15
- minitap/mobile_use/tools/mobile/swipe.py +3 -26
- minitap/mobile_use/tools/mobile/tap.py +41 -28
- minitap/mobile_use/tools/mobile/wait_for_delay.py +84 -0
- minitap/mobile_use/utils/cli_helpers.py +10 -6
- {minitap_mobile_use-2.5.3.dist-info → minitap_mobile_use-2.6.0.dist-info}/METADATA +1 -1
- {minitap_mobile_use-2.5.3.dist-info → minitap_mobile_use-2.6.0.dist-info}/RECORD +38 -36
- minitap/mobile_use/tools/mobile/glimpse_screen.py +0 -74
- minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +0 -64
- {minitap_mobile_use-2.5.3.dist-info → minitap_mobile_use-2.6.0.dist-info}/WHEEL +0 -0
- {minitap_mobile_use-2.5.3.dist-info → minitap_mobile_use-2.6.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,41 +1,44 @@
|
|
|
1
1
|
minitap/mobile_use/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
|
|
2
|
-
minitap/mobile_use/agents/contextor/contextor.py,sha256=
|
|
3
|
-
minitap/mobile_use/agents/cortex/cortex.md,sha256=
|
|
4
|
-
minitap/mobile_use/agents/cortex/cortex.py,sha256=
|
|
5
|
-
minitap/mobile_use/agents/cortex/types.py,sha256=
|
|
6
|
-
minitap/mobile_use/agents/executor/executor.md,sha256=
|
|
7
|
-
minitap/mobile_use/agents/executor/executor.py,sha256=
|
|
2
|
+
minitap/mobile_use/agents/contextor/contextor.py,sha256=6c40c226a08219eaac34e763931d3cca2b4b2f3650afe09fd1b5f37880647969,1357
|
|
3
|
+
minitap/mobile_use/agents/cortex/cortex.md,sha256=5ecc65b221f93f4b6daf50a88936bb0e585a7213ddca12e30710754f55a65f84,16321
|
|
4
|
+
minitap/mobile_use/agents/cortex/cortex.py,sha256=21447e151a34386ec2a260ad70b0fb6acf5bb2bf4f8ccd54028bb531b935be02,5831
|
|
5
|
+
minitap/mobile_use/agents/cortex/types.py,sha256=04c4c91f55a587b0c3e612243d7bf1fca7328f65a7d13999df4708d6a81b54dc,945
|
|
6
|
+
minitap/mobile_use/agents/executor/executor.md,sha256=a7b922203cc80b068e6a0a79b298757c8a11f45e1add84c04f026e4c351336ea,3756
|
|
7
|
+
minitap/mobile_use/agents/executor/executor.py,sha256=99c84af73685c57d56db12874a60208619b5c329b291cc91ecbcd1a832130680,3369
|
|
8
8
|
minitap/mobile_use/agents/executor/tool_node.py,sha256=2ad729ede393882460ae3d180ac1c0e1ab1688f40b2017220aad1b059f6485c5,3900
|
|
9
9
|
minitap/mobile_use/agents/executor/utils.py,sha256=74cf2287053cd4fc763835870e5ae029ca36f6cd0ca7d50678ad05d52ab265b7,368
|
|
10
|
-
minitap/mobile_use/agents/hopper/hopper.md,sha256=
|
|
11
|
-
minitap/mobile_use/agents/hopper/hopper.py,sha256=
|
|
10
|
+
minitap/mobile_use/agents/hopper/hopper.md,sha256=a1764bc43c9b53b3bfe0584abbf0ff7b6e3f516faab3f368fa27bfd9090b5770,1555
|
|
11
|
+
minitap/mobile_use/agents/hopper/hopper.py,sha256=76d87ba79ce9132a54224a0d8380d05d8947c292b40528ed67381fa0bfc14b4c,1650
|
|
12
12
|
minitap/mobile_use/agents/orchestrator/human.md,sha256=6559026aa921b7ad7dddcf3dfcd5d9930252edd6484d60ea92ff6ca97ed028fc,229
|
|
13
13
|
minitap/mobile_use/agents/orchestrator/orchestrator.md,sha256=cc1a353c577f2eef42d9a528178ccc2c0a4a144a907a29878f9efe57e83b12fa,2546
|
|
14
|
-
minitap/mobile_use/agents/orchestrator/orchestrator.py,sha256=
|
|
14
|
+
minitap/mobile_use/agents/orchestrator/orchestrator.py,sha256=21af0a1771ecc0880b7d2238c707c37dd57351510089d39139c5d6fdc34b0be1,5503
|
|
15
15
|
minitap/mobile_use/agents/orchestrator/types.py,sha256=f53dfdc99e8d50888ac1cde5f7f90ba5c87837a8eee8dd8efa31f2640394433c,335
|
|
16
16
|
minitap/mobile_use/agents/outputter/human.md,sha256=6b9b45c640b163554524b1aec4cd97134c628eeb8557a32e23c8f966d32f642e,771
|
|
17
|
-
minitap/mobile_use/agents/outputter/outputter.py,sha256=
|
|
17
|
+
minitap/mobile_use/agents/outputter/outputter.py,sha256=d6e788882bf63b2ac48823575d561ad4a43d65b584b81626d24a8a8c56ef863f,3331
|
|
18
18
|
minitap/mobile_use/agents/outputter/test_outputter.py,sha256=907b517c486f82a384a364e0bd202c00e8e8082c138461f6eead0e25c2779ba9,5538
|
|
19
19
|
minitap/mobile_use/agents/planner/human.md,sha256=cb37be2af568918e60238eaa785837178a3ba8f8112de86850d9a62914c18314,222
|
|
20
|
-
minitap/mobile_use/agents/planner/planner.md,sha256=
|
|
21
|
-
minitap/mobile_use/agents/planner/planner.py,sha256=
|
|
20
|
+
minitap/mobile_use/agents/planner/planner.md,sha256=6c003612c7ad2fece469e6f439751643a69e6559e604d1f8a5e14663fae04a4b,4527
|
|
21
|
+
minitap/mobile_use/agents/planner/planner.py,sha256=c4cae78cb2a4d9f3fe3b386e2c04d560ff08c8ce26474feeb93d1419990cdf8e,3294
|
|
22
22
|
minitap/mobile_use/agents/planner/types.py,sha256=e8acf3c2d1505286a138b6f7c3ef36f397d154953311d0875e3ef35152653e7f,1496
|
|
23
23
|
minitap/mobile_use/agents/planner/utils.py,sha256=88b4b039e09cea254615ff3d0bc8951c3717a68702742e913a0176ecfbcaf495,2315
|
|
24
|
+
minitap/mobile_use/agents/screen_analyzer/human.md,sha256=1421b0cf68507ebc2c4df6b76e4d4e738b9ebee4d4691cc8d623680b6f33dfef,352
|
|
25
|
+
minitap/mobile_use/agents/screen_analyzer/screen_analyzer.py,sha256=38d88206d1d0ee3917afe88760cf181a4f9d5b6a0e7e03de619d739959924c78,4069
|
|
24
26
|
minitap/mobile_use/agents/summarizer/summarizer.py,sha256=56c2c7d5d48f4ba045b1401538db78b2ddbd43280389ed4cbc58ecd1da0c7610,1083
|
|
25
27
|
minitap/mobile_use/clients/device_hardware_client.py,sha256=9593380a7a3df32f02aa22717678c25e91367df26b1743abde9e57aec5dc2474,857
|
|
26
|
-
minitap/mobile_use/clients/ios_client.py,sha256=
|
|
28
|
+
minitap/mobile_use/clients/ios_client.py,sha256=74b417bbfba597a50c534d76065aa160efdf3810a16fd5c1f74323a4a705fcab,1629
|
|
27
29
|
minitap/mobile_use/clients/screen_api_client.py,sha256=3615dc65d25c38b4d8dc5512f9adb3bcf69dca7a0298a472a6812f604a275c47,2019
|
|
28
|
-
minitap/mobile_use/config.py,sha256=
|
|
30
|
+
minitap/mobile_use/config.py,sha256=82ff1a57e66d86156efa7112ceeef6f710d22b69fa4e883e7e864e6f5707f746,13584
|
|
29
31
|
minitap/mobile_use/constants.py,sha256=3acd9d6ade5bc772e902b3473f3ba12ddd04e7306963ca2bae49d1132d89ba46,95
|
|
30
32
|
minitap/mobile_use/context.py,sha256=2e212588a8a7caf58460fead0cae195c6386b7301026bbb2d78fe56c263ce6c3,2131
|
|
31
33
|
minitap/mobile_use/controllers/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
|
|
32
|
-
minitap/mobile_use/controllers/mobile_command_controller.py,sha256=
|
|
33
|
-
minitap/mobile_use/controllers/platform_specific_commands_controller.py,sha256=
|
|
34
|
-
minitap/mobile_use/
|
|
35
|
-
minitap/mobile_use/graph/
|
|
36
|
-
minitap/mobile_use/
|
|
34
|
+
minitap/mobile_use/controllers/mobile_command_controller.py,sha256=e1eadfa0d4555a04174baa520b7c8f01ff83d1c8cbc90edf4c7052408cec337b,20361
|
|
35
|
+
minitap/mobile_use/controllers/platform_specific_commands_controller.py,sha256=84061d2a44ad8eed985f339866a4eb9456d7b9007ecf2ce6277e08335cf8f564,3274
|
|
36
|
+
minitap/mobile_use/controllers/types.py,sha256=c4dd6b266dd8f157ca1e6a211369ba8e7f65d3fda72b0742bda1128eefd99473,2935
|
|
37
|
+
minitap/mobile_use/graph/graph.py,sha256=24eab15a3540a6b63004eb82ca3577cd1da3f83354a9a49bb3e7a0c7b67f64df,5797
|
|
38
|
+
minitap/mobile_use/graph/state.py,sha256=1903f49772be7f9725e69718194a7be953203f882ec5e26211a5db6fdc569478,3596
|
|
39
|
+
minitap/mobile_use/main.py,sha256=1405a13eab2c3b86b148bb9678e829447522753a10572935fc1e3ed3bbce6878,3927
|
|
37
40
|
minitap/mobile_use/sdk/__init__.py,sha256=4e5555c0597242b9523827194a2500b9c6d7e5c04b1ccd2056c9b1f4d42a31cd,318
|
|
38
|
-
minitap/mobile_use/sdk/agent.py,sha256=
|
|
41
|
+
minitap/mobile_use/sdk/agent.py,sha256=228fdba23e29794d878a42d598bf0ed3b8af5a8a4d1526d57a53966f24773d11,30345
|
|
39
42
|
minitap/mobile_use/sdk/builders/__init__.py,sha256=d6c96d39b80900a114698ef205ab5061a541f33bfa99c456d9345e5adb8ff6ff,424
|
|
40
43
|
minitap/mobile_use/sdk/builders/agent_config_builder.py,sha256=542d7d06c677059c8f8e0bc27d95f2dce282b1cfe0320fa6f902bc2f9d08140d,7686
|
|
41
44
|
minitap/mobile_use/sdk/builders/index.py,sha256=64336ac3b3dea4673a48e95b8c5ac4196ecd5d2196380377d102593d0a1dc138,442
|
|
@@ -46,7 +49,7 @@ minitap/mobile_use/sdk/examples/__init__.py,sha256=c23868a2ca7e9b76e80d6835fe93c
|
|
|
46
49
|
minitap/mobile_use/sdk/examples/platform_manual_task_example.py,sha256=00f54d58fa0abe9a1df20b3593633c239947c5acdf716fe9be1f58f9f56d8caa,1937
|
|
47
50
|
minitap/mobile_use/sdk/examples/platform_minimal_example.py,sha256=bdb86142f4bb5d95d54838fc6487bb06e5e21db5106385344138be1653071aea,1389
|
|
48
51
|
minitap/mobile_use/sdk/examples/simple_photo_organizer.py,sha256=8ad1cebb5281e3663264560bd15b090add41d2821b1db77e4cbc829860c98df8,2606
|
|
49
|
-
minitap/mobile_use/sdk/examples/smart_notification_assistant.py,sha256=
|
|
52
|
+
minitap/mobile_use/sdk/examples/smart_notification_assistant.py,sha256=ecab26c6fd2af59bb22dbc67f0eeb4a4a73e4394639b34fb2d44f04f218e2b61,8104
|
|
50
53
|
minitap/mobile_use/sdk/services/platform.py,sha256=2a1f97efe06577b67c5880ebcfb8a855ba61bf30b5f7d3e1f2600ef60cbd9dce,11969
|
|
51
54
|
minitap/mobile_use/sdk/types/__init__.py,sha256=433aff6b35f84a985633204edbbdaca9f2f61fb2b822630f9723c481b9bb5c10,1078
|
|
52
55
|
minitap/mobile_use/sdk/types/agent.py,sha256=390d5c642b3480f4a2203ddd28ec115c785f2576bec81e82e4db3c129399c020,2260
|
|
@@ -55,32 +58,31 @@ minitap/mobile_use/sdk/types/platform.py,sha256=df68f9b158542eb4d31c7151abf638a7
|
|
|
55
58
|
minitap/mobile_use/sdk/types/task.py,sha256=264e77bce958ee398252c7346dfa7ae896ea6dbbd98576f35d1a766c4926a189,8236
|
|
56
59
|
minitap/mobile_use/sdk/utils.py,sha256=647f1f4a463c3029c3b0eb3c33f7dd778d5f5fd9d293224f5474595a60e1de6f,967
|
|
57
60
|
minitap/mobile_use/servers/config.py,sha256=8a4a6bce23e2093d047a91e135e2f88627f76ac12177d071f25a3ca739b3afeb,575
|
|
58
|
-
minitap/mobile_use/servers/device_hardware_bridge.py,sha256=
|
|
61
|
+
minitap/mobile_use/servers/device_hardware_bridge.py,sha256=39c20834812d9929163affaedd0e285ab0b349948b3236156c04a3e0bf094272,7456
|
|
59
62
|
minitap/mobile_use/servers/device_screen_api.py,sha256=2e60a5bc32d71ca80ee8dd93ff8b76138ae5d74e67d82bdb16fd15086d4eb33a,5675
|
|
60
63
|
minitap/mobile_use/servers/start_servers.py,sha256=2155d744726ac64c38b38e2e589e1cdb32e852bd9d5cc8c669f175d79de0a78b,5078
|
|
61
64
|
minitap/mobile_use/servers/stop_servers.py,sha256=04a409a17fc0323209301fe28fbb037d71e41e5422eb369640f9f329aae312f5,7064
|
|
62
65
|
minitap/mobile_use/servers/utils.py,sha256=f3cc85da39f8d60cb840001be418562de7db95462370db9b79e96d884abe5c17,294
|
|
63
66
|
minitap/mobile_use/services/accessibility.py,sha256=42bcbe81b427ee6f6e82bcfe420fc40630db950bda354e3e433c2dda2e159628,3404
|
|
64
|
-
minitap/mobile_use/services/llm.py,sha256=
|
|
65
|
-
minitap/mobile_use/tools/index.py,sha256=
|
|
67
|
+
minitap/mobile_use/services/llm.py,sha256=7d8c784085fb4f91a1b5d2e715f07e89a686fbdf7aeba4afda444c63ac9b43d3,7153
|
|
68
|
+
minitap/mobile_use/tools/index.py,sha256=3b09ac41efb38a339ed4dd265c37aca85bd56f26938df45a6723b9b06071a3ca,2294
|
|
66
69
|
minitap/mobile_use/tools/mobile/back.py,sha256=8b909e412c8ad382e339b8c89171bf1363398a95a5c96126f79ee2f24e3c2ed1,1816
|
|
67
|
-
minitap/mobile_use/tools/mobile/clear_text.py,sha256=58e0426f042e39d827d3ff2fe3a79bb23e4146cbefd13da938e1a6f397579c4b,9799
|
|
68
70
|
minitap/mobile_use/tools/mobile/erase_one_char.py,sha256=a125933619614b621479c514c8e29a7ec6b504b1e7ab1be13603a744f32322d6,1985
|
|
69
|
-
minitap/mobile_use/tools/mobile/
|
|
70
|
-
minitap/mobile_use/tools/mobile/
|
|
71
|
+
minitap/mobile_use/tools/mobile/focus_and_clear_text.py,sha256=a1a8424cd3c3776b9dc771a05dfa8f0dfbd7159e48146fe38466f9fc0af1840e,9869
|
|
72
|
+
minitap/mobile_use/tools/mobile/focus_and_input_text.py,sha256=ac2126c8b6451f3e7b4864b214042d3fcc0886b2ce62478c93d42e1d75f7893b,5507
|
|
71
73
|
minitap/mobile_use/tools/mobile/launch_app.py,sha256=a794665f9bc7a035ad6029b8153d6e56a0ba49ce36b0ab8a4e342267465d8fef,3237
|
|
72
|
-
minitap/mobile_use/tools/mobile/long_press_on.py,sha256=
|
|
74
|
+
minitap/mobile_use/tools/mobile/long_press_on.py,sha256=5103dd05b25b7f8633c52a103d863ccfb6539273a341ca92398a03c0652ff1fa,7791
|
|
73
75
|
minitap/mobile_use/tools/mobile/open_link.py,sha256=3017b94a921ced032f18526101eeba8caa907ebee40b82bed0baa4f35477e6c8,1980
|
|
74
76
|
minitap/mobile_use/tools/mobile/press_key.py,sha256=0133b1f1eb3b67c94aeaa70ac3ff822dc1e1f70624a0727f6e8c3cbcef1a80ab,2044
|
|
75
77
|
minitap/mobile_use/tools/mobile/stop_app.py,sha256=74f93c60cdf039455897b0bb2d74e5702cc8429abc606b6069c7be6399b7fbdf,2166
|
|
76
|
-
minitap/mobile_use/tools/mobile/swipe.py,sha256=
|
|
77
|
-
minitap/mobile_use/tools/mobile/tap.py,sha256=
|
|
78
|
-
minitap/mobile_use/tools/mobile/
|
|
78
|
+
minitap/mobile_use/tools/mobile/swipe.py,sha256=a882d8f5924c118b750fdd3e82019bf5bb205a3ad165afad3ed7e3466530bb3b,5062
|
|
79
|
+
minitap/mobile_use/tools/mobile/tap.py,sha256=9346060e33c55bc30c7e02fbf50cfbfca39fec295f804ff15365ca5cb782808c,6446
|
|
80
|
+
minitap/mobile_use/tools/mobile/wait_for_delay.py,sha256=19131b7f0c842cd03886e6ae2b573c210c0ca0b266b68478fd349efccd8888b7,2846
|
|
79
81
|
minitap/mobile_use/tools/test_utils.py,sha256=42dd0ff789c92254bc4452ab339a53f0d597a9bfddd2a6554d37f54ff52920ef,13888
|
|
80
82
|
minitap/mobile_use/tools/tool_wrapper.py,sha256=f0f27beaae25a1bcfd9b72bf994de84b2e5fba9e242d8ad18a8d1a97cd7619e4,454
|
|
81
83
|
minitap/mobile_use/tools/types.py,sha256=4f73939260509269416f5ad515a34569f7fa3467e1f920b9e8a991d1adbc761a,1298
|
|
82
84
|
minitap/mobile_use/tools/utils.py,sha256=7a80b9f82447a718ecc1fd2163f13f35f4f0b4e2859abd85f3c014ded10bccf2,6926
|
|
83
|
-
minitap/mobile_use/utils/cli_helpers.py,sha256=
|
|
85
|
+
minitap/mobile_use/utils/cli_helpers.py,sha256=c3fb6949f167200d1b76b8655da51dfe4b6f944bf4fb49bbb6b1d9945719b302,1687
|
|
84
86
|
minitap/mobile_use/utils/cli_selection.py,sha256=62e949bf075e984b5d23b4a9880ff2bccf8f9e0f7ccb48120030a6a82075352b,4788
|
|
85
87
|
minitap/mobile_use/utils/conversations.py,sha256=8f1d924300ec3f6f7c71510c21e3011b75caca5b1fff06fdaccb377c3cde24ec,914
|
|
86
88
|
minitap/mobile_use/utils/decorators.py,sha256=0bb30fb4f5d5cef0aef45643e68e199d39910f1d771eb4086f3e083d566c16a5,3591
|
|
@@ -94,7 +96,7 @@ minitap/mobile_use/utils/shell_utils.py,sha256=b35ae7f863379adb86c9ba0f9b3b9d495
|
|
|
94
96
|
minitap/mobile_use/utils/test_ui_hierarchy.py,sha256=96c1549c05b4f7254a22d57dbd40aea860756f1e0b9d8cc24319383643448422,5911
|
|
95
97
|
minitap/mobile_use/utils/time.py,sha256=41bfaabb3751de11443ccb4a3f1f53d5ebacc7744c72e32695fdcc3d23f17d49,160
|
|
96
98
|
minitap/mobile_use/utils/ui_hierarchy.py,sha256=f3370518035d9daf02c08042a9e28ad564f4fc81a2b268103b9a7f8bc5c61d11,3797
|
|
97
|
-
minitap_mobile_use-2.
|
|
98
|
-
minitap_mobile_use-2.
|
|
99
|
-
minitap_mobile_use-2.
|
|
100
|
-
minitap_mobile_use-2.
|
|
99
|
+
minitap_mobile_use-2.6.0.dist-info/WHEEL,sha256=ab6157bc637547491fb4567cd7ddf26b04d63382916ca16c29a5c8e94c9c9ef7,79
|
|
100
|
+
minitap_mobile_use-2.6.0.dist-info/entry_points.txt,sha256=663a29cfd551a4eaa0f27335f0bd7e4a732a4e39c76b68ef5c8dc444d4a285fa,60
|
|
101
|
+
minitap_mobile_use-2.6.0.dist-info/METADATA,sha256=a6c9faf5ffdd6b6710f3889953f4f4aae31a131c0d2a3ce6e0a1a3d4f5c7e1f8,11995
|
|
102
|
+
minitap_mobile_use-2.6.0.dist-info/RECORD,,
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
from typing import Annotated
|
|
2
|
-
|
|
3
|
-
from langchain_core.messages import ToolMessage
|
|
4
|
-
from langchain_core.tools import tool
|
|
5
|
-
from langchain_core.tools.base import InjectedToolCallId
|
|
6
|
-
from langgraph.prebuilt import InjectedState
|
|
7
|
-
from langgraph.types import Command
|
|
8
|
-
|
|
9
|
-
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
10
|
-
from minitap.mobile_use.context import MobileUseContext
|
|
11
|
-
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
12
|
-
take_screenshot as take_screenshot_controller,
|
|
13
|
-
)
|
|
14
|
-
from minitap.mobile_use.graph.state import State
|
|
15
|
-
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
16
|
-
from minitap.mobile_use.utils.media import compress_base64_jpeg
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def get_glimpse_screen_tool(ctx: MobileUseContext):
|
|
20
|
-
@tool
|
|
21
|
-
async def glimpse_screen(
|
|
22
|
-
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
23
|
-
state: Annotated[State, InjectedState],
|
|
24
|
-
agent_thought: str,
|
|
25
|
-
) -> Command:
|
|
26
|
-
"""
|
|
27
|
-
Captures the current screen as an image.
|
|
28
|
-
The resulting screenshot is added to the context for the next reasoning step.
|
|
29
|
-
"""
|
|
30
|
-
compressed_image_base64 = None
|
|
31
|
-
has_failed = False
|
|
32
|
-
|
|
33
|
-
try:
|
|
34
|
-
output = take_screenshot_controller(ctx=ctx)
|
|
35
|
-
compressed_image_base64 = compress_base64_jpeg(output)
|
|
36
|
-
except Exception as e:
|
|
37
|
-
output = str(e)
|
|
38
|
-
has_failed = True
|
|
39
|
-
|
|
40
|
-
agent_outcome = (
|
|
41
|
-
glimpse_screen_wrapper.on_failure_fn()
|
|
42
|
-
if has_failed
|
|
43
|
-
else glimpse_screen_wrapper.on_success_fn()
|
|
44
|
-
)
|
|
45
|
-
|
|
46
|
-
tool_message = ToolMessage(
|
|
47
|
-
tool_call_id=tool_call_id,
|
|
48
|
-
content=agent_outcome,
|
|
49
|
-
additional_kwargs={"error": output} if has_failed else {},
|
|
50
|
-
status="error" if has_failed else "success",
|
|
51
|
-
)
|
|
52
|
-
updates = {
|
|
53
|
-
"agents_thoughts": [agent_thought, agent_outcome],
|
|
54
|
-
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
55
|
-
}
|
|
56
|
-
if compressed_image_base64:
|
|
57
|
-
updates["latest_screenshot_base64"] = compressed_image_base64
|
|
58
|
-
return Command(
|
|
59
|
-
update=await state.asanitize_update(
|
|
60
|
-
ctx=ctx,
|
|
61
|
-
update=updates,
|
|
62
|
-
agent="executor",
|
|
63
|
-
),
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
return glimpse_screen
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
glimpse_screen_wrapper = ToolWrapper(
|
|
70
|
-
tool_fn_getter=get_glimpse_screen_tool,
|
|
71
|
-
on_success_fn=lambda: "Visual context captured successfully."
|
|
72
|
-
+ "It is now available for immediate analysis.",
|
|
73
|
-
on_failure_fn=lambda: "Failed to capture visual context.",
|
|
74
|
-
)
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
from langchain_core.messages import ToolMessage
|
|
2
|
-
from langchain_core.tools import tool
|
|
3
|
-
from langchain_core.tools.base import InjectedToolCallId
|
|
4
|
-
from langgraph.prebuilt import InjectedState
|
|
5
|
-
from langgraph.types import Command
|
|
6
|
-
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
7
|
-
from minitap.mobile_use.context import MobileUseContext
|
|
8
|
-
from minitap.mobile_use.controllers.mobile_command_controller import WaitTimeout
|
|
9
|
-
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
10
|
-
wait_for_animation_to_end as wait_for_animation_to_end_controller,
|
|
11
|
-
)
|
|
12
|
-
from minitap.mobile_use.graph.state import State
|
|
13
|
-
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
14
|
-
from typing import Annotated
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
|
|
18
|
-
@tool
|
|
19
|
-
async def wait_for_animation_to_end(
|
|
20
|
-
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
|
-
state: Annotated[State, InjectedState],
|
|
22
|
-
agent_thought: str,
|
|
23
|
-
timeout: WaitTimeout | None,
|
|
24
|
-
) -> Command:
|
|
25
|
-
"""
|
|
26
|
-
Waits for ongoing animations or videos to finish before continuing.
|
|
27
|
-
|
|
28
|
-
If a `timeout` (in milliseconds) is set, the command proceeds after the timeout even if
|
|
29
|
-
the animation hasn't ended.
|
|
30
|
-
The flow continues immediately once the animation is detected as complete.
|
|
31
|
-
|
|
32
|
-
Example:
|
|
33
|
-
- waitForAnimationToEnd
|
|
34
|
-
- waitForAnimationToEnd: { timeout: 5000 }
|
|
35
|
-
"""
|
|
36
|
-
output = wait_for_animation_to_end_controller(ctx=ctx, timeout=timeout)
|
|
37
|
-
has_failed = output is not None
|
|
38
|
-
tool_message = ToolMessage(
|
|
39
|
-
tool_call_id=tool_call_id,
|
|
40
|
-
content=wait_for_animation_to_end_wrapper.on_failure_fn()
|
|
41
|
-
if has_failed
|
|
42
|
-
else wait_for_animation_to_end_wrapper.on_success_fn(timeout),
|
|
43
|
-
additional_kwargs={"error": output} if has_failed else {},
|
|
44
|
-
status="error" if has_failed else "success",
|
|
45
|
-
)
|
|
46
|
-
return Command(
|
|
47
|
-
update=await state.asanitize_update(
|
|
48
|
-
ctx=ctx,
|
|
49
|
-
update={
|
|
50
|
-
"agents_thoughts": [agent_thought],
|
|
51
|
-
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
52
|
-
},
|
|
53
|
-
agent="executor",
|
|
54
|
-
),
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
return wait_for_animation_to_end
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
wait_for_animation_to_end_wrapper = ToolWrapper(
|
|
61
|
-
tool_fn_getter=get_wait_for_animation_to_end_tool,
|
|
62
|
-
on_success_fn=lambda: "Animation ended successfully.",
|
|
63
|
-
on_failure_fn=lambda: "Failed to end animation.",
|
|
64
|
-
)
|
|
File without changes
|
|
File without changes
|