minitap-mobile-use 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of minitap-mobile-use might be problematic. Click here for more details.
- minitap/mobile_use/agents/cortex/cortex.md +19 -10
- minitap/mobile_use/agents/cortex/cortex.py +15 -2
- minitap/mobile_use/agents/cortex/types.py +2 -4
- minitap/mobile_use/agents/executor/executor.md +20 -15
- minitap/mobile_use/agents/executor/executor.py +6 -18
- minitap/mobile_use/agents/executor/tool_node.py +105 -0
- minitap/mobile_use/agents/hopper/hopper.md +2 -10
- minitap/mobile_use/agents/hopper/hopper.py +4 -9
- minitap/mobile_use/agents/orchestrator/human.md +3 -4
- minitap/mobile_use/agents/orchestrator/orchestrator.md +25 -7
- minitap/mobile_use/agents/orchestrator/orchestrator.py +56 -56
- minitap/mobile_use/agents/orchestrator/types.py +5 -8
- minitap/mobile_use/agents/outputter/outputter.py +1 -2
- minitap/mobile_use/agents/planner/planner.md +25 -15
- minitap/mobile_use/agents/planner/planner.py +7 -1
- minitap/mobile_use/agents/planner/types.py +10 -5
- minitap/mobile_use/agents/planner/utils.py +11 -0
- minitap/mobile_use/agents/summarizer/summarizer.py +2 -1
- minitap/mobile_use/clients/device_hardware_client.py +3 -0
- minitap/mobile_use/config.py +16 -14
- minitap/mobile_use/constants.py +1 -0
- minitap/mobile_use/context.py +3 -4
- minitap/mobile_use/controllers/mobile_command_controller.py +37 -26
- minitap/mobile_use/controllers/platform_specific_commands_controller.py +3 -4
- minitap/mobile_use/graph/graph.py +10 -31
- minitap/mobile_use/graph/state.py +34 -14
- minitap/mobile_use/main.py +11 -8
- minitap/mobile_use/sdk/agent.py +78 -63
- minitap/mobile_use/sdk/builders/agent_config_builder.py +23 -11
- minitap/mobile_use/sdk/builders/task_request_builder.py +9 -9
- minitap/mobile_use/sdk/examples/smart_notification_assistant.py +1 -2
- minitap/mobile_use/sdk/types/agent.py +10 -5
- minitap/mobile_use/sdk/types/task.py +19 -18
- minitap/mobile_use/sdk/utils.py +1 -1
- minitap/mobile_use/servers/config.py +1 -2
- minitap/mobile_use/servers/device_hardware_bridge.py +3 -4
- minitap/mobile_use/servers/start_servers.py +4 -4
- minitap/mobile_use/servers/stop_servers.py +12 -18
- minitap/mobile_use/services/llm.py +4 -2
- minitap/mobile_use/tools/index.py +11 -7
- minitap/mobile_use/tools/mobile/back.py +8 -12
- minitap/mobile_use/tools/mobile/clear_text.py +277 -0
- minitap/mobile_use/tools/mobile/copy_text_from.py +8 -12
- minitap/mobile_use/tools/mobile/erase_one_char.py +56 -0
- minitap/mobile_use/tools/mobile/find_packages.py +69 -0
- minitap/mobile_use/tools/mobile/input_text.py +55 -32
- minitap/mobile_use/tools/mobile/launch_app.py +8 -12
- minitap/mobile_use/tools/mobile/long_press_on.py +9 -13
- minitap/mobile_use/tools/mobile/open_link.py +8 -12
- minitap/mobile_use/tools/mobile/paste_text.py +8 -12
- minitap/mobile_use/tools/mobile/press_key.py +8 -12
- minitap/mobile_use/tools/mobile/stop_app.py +9 -13
- minitap/mobile_use/tools/mobile/swipe.py +8 -12
- minitap/mobile_use/tools/mobile/take_screenshot.py +8 -12
- minitap/mobile_use/tools/mobile/tap.py +9 -13
- minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +9 -13
- minitap/mobile_use/tools/tool_wrapper.py +1 -23
- minitap/mobile_use/tools/utils.py +86 -0
- minitap/mobile_use/utils/cli_helpers.py +1 -2
- minitap/mobile_use/utils/cli_selection.py +5 -6
- minitap/mobile_use/utils/decorators.py +21 -20
- minitap/mobile_use/utils/logger.py +3 -4
- minitap/mobile_use/utils/media.py +1 -1
- minitap/mobile_use/utils/recorder.py +11 -10
- minitap/mobile_use/utils/ui_hierarchy.py +98 -3
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/METADATA +12 -2
- minitap_mobile_use-2.1.0.dist-info/RECORD +96 -0
- minitap/mobile_use/agents/executor/executor_context_cleaner.py +0 -27
- minitap/mobile_use/tools/mobile/erase_text.py +0 -124
- minitap/mobile_use/tools/mobile/list_packages.py +0 -78
- minitap/mobile_use/tools/mobile/run_flow.py +0 -57
- minitap_mobile_use-2.0.0.dist-info/RECORD +0 -95
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/WHEEL +0 -0
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.types import Command
|
|
5
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
7
6
|
from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
|
|
8
7
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
9
8
|
copy_text_from as copy_text_from_controller,
|
|
10
9
|
)
|
|
11
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
10
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
12
11
|
from pydantic import Field
|
|
13
|
-
from
|
|
12
|
+
from typing import Annotated
|
|
14
13
|
from minitap.mobile_use.context import MobileUseContext
|
|
15
14
|
from minitap.mobile_use.graph.state import State
|
|
16
15
|
from langgraph.prebuilt import InjectedState
|
|
@@ -22,7 +21,6 @@ def get_copy_text_from_tool(ctx: MobileUseContext):
|
|
|
22
21
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
23
22
|
state: Annotated[State, InjectedState],
|
|
24
23
|
agent_thought: str,
|
|
25
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
26
24
|
selector_request: SelectorRequest = Field(
|
|
27
25
|
..., description="The selector to copy text from"
|
|
28
26
|
),
|
|
@@ -50,18 +48,16 @@ def get_copy_text_from_tool(ctx: MobileUseContext):
|
|
|
50
48
|
if has_failed
|
|
51
49
|
else copy_text_from_wrapper.on_success_fn(selector_request),
|
|
52
50
|
additional_kwargs={"error": output} if has_failed else {},
|
|
51
|
+
status="error" if has_failed else "success",
|
|
53
52
|
)
|
|
54
53
|
return Command(
|
|
55
|
-
update=
|
|
54
|
+
update=state.sanitize_update(
|
|
56
55
|
ctx=ctx,
|
|
57
|
-
|
|
58
|
-
executor_metadata=executor_metadata,
|
|
59
|
-
tool_message=tool_message,
|
|
60
|
-
is_failure=has_failed,
|
|
61
|
-
updates={
|
|
56
|
+
update={
|
|
62
57
|
"agents_thoughts": [agent_thought],
|
|
63
|
-
|
|
58
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
64
59
|
},
|
|
60
|
+
agent="executor",
|
|
65
61
|
),
|
|
66
62
|
)
|
|
67
63
|
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from langchain_core.messages import ToolMessage
|
|
2
|
+
from langchain_core.tools import tool
|
|
3
|
+
from langchain_core.tools.base import InjectedToolCallId
|
|
4
|
+
from langgraph.prebuilt import InjectedState
|
|
5
|
+
from langgraph.types import Command
|
|
6
|
+
from typing import Annotated
|
|
7
|
+
|
|
8
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
9
|
+
from minitap.mobile_use.context import MobileUseContext
|
|
10
|
+
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
11
|
+
erase_text as erase_text_controller,
|
|
12
|
+
)
|
|
13
|
+
from minitap.mobile_use.graph.state import State
|
|
14
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_erase_one_char_tool(ctx: MobileUseContext):
|
|
18
|
+
@tool
|
|
19
|
+
def erase_one_char(
|
|
20
|
+
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
|
+
state: Annotated[State, InjectedState],
|
|
22
|
+
agent_thought: str,
|
|
23
|
+
):
|
|
24
|
+
"""
|
|
25
|
+
Erase one character from a text area.
|
|
26
|
+
It acts the same as pressing backspace a single time.
|
|
27
|
+
"""
|
|
28
|
+
output = erase_text_controller(ctx=ctx, nb_chars=1)
|
|
29
|
+
has_failed = output is not None
|
|
30
|
+
tool_message = ToolMessage(
|
|
31
|
+
tool_call_id=tool_call_id,
|
|
32
|
+
content=erase_one_char_wrapper.on_failure_fn()
|
|
33
|
+
if has_failed
|
|
34
|
+
else erase_one_char_wrapper.on_success_fn(),
|
|
35
|
+
additional_kwargs={"error": output} if has_failed else {},
|
|
36
|
+
status="error" if has_failed else "success",
|
|
37
|
+
)
|
|
38
|
+
return Command(
|
|
39
|
+
update=state.sanitize_update(
|
|
40
|
+
ctx=ctx,
|
|
41
|
+
update={
|
|
42
|
+
"agents_thoughts": [agent_thought],
|
|
43
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
44
|
+
},
|
|
45
|
+
agent="executor",
|
|
46
|
+
),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
return erase_one_char
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
erase_one_char_wrapper = ToolWrapper(
|
|
53
|
+
tool_fn_getter=get_erase_one_char_tool,
|
|
54
|
+
on_success_fn=lambda: "Erased one character successfully.",
|
|
55
|
+
on_failure_fn=lambda: "Failed to erase one character.",
|
|
56
|
+
)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from langchain_core.messages import ToolMessage
|
|
2
|
+
from langchain_core.tools import tool
|
|
3
|
+
from langchain_core.tools.base import InjectedToolCallId
|
|
4
|
+
from langgraph.prebuilt import InjectedState
|
|
5
|
+
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.agents.hopper.hopper import HopperOutput, hopper
|
|
7
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
|
+
from minitap.mobile_use.context import MobileUseContext
|
|
9
|
+
from minitap.mobile_use.controllers.platform_specific_commands_controller import list_packages
|
|
10
|
+
from minitap.mobile_use.graph.state import State
|
|
11
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
12
|
+
from typing import Annotated
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_find_packages_tool(ctx: MobileUseContext):
|
|
16
|
+
@tool
|
|
17
|
+
async def find_packages(
|
|
18
|
+
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
19
|
+
state: Annotated[State, InjectedState],
|
|
20
|
+
appNames: list[str],
|
|
21
|
+
agent_thought: str,
|
|
22
|
+
) -> Command:
|
|
23
|
+
"""
|
|
24
|
+
Finds relevant applications.
|
|
25
|
+
Outputs the full package names list (android) or bundle ids list (IOS).
|
|
26
|
+
"""
|
|
27
|
+
output: str = list_packages(ctx=ctx)
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
hopper_output: HopperOutput = await hopper(
|
|
31
|
+
ctx=ctx,
|
|
32
|
+
request=f"I'm looking for the package names of the following apps: {appNames}",
|
|
33
|
+
data=output,
|
|
34
|
+
)
|
|
35
|
+
tool_message = ToolMessage(
|
|
36
|
+
tool_call_id=tool_call_id,
|
|
37
|
+
content=find_packages_wrapper.on_success_fn(
|
|
38
|
+
hopper_output.step, hopper_output.output
|
|
39
|
+
),
|
|
40
|
+
status="success",
|
|
41
|
+
)
|
|
42
|
+
except Exception as e:
|
|
43
|
+
print("Failed to extract insights from data: " + str(e))
|
|
44
|
+
tool_message = ToolMessage(
|
|
45
|
+
tool_call_id=tool_call_id,
|
|
46
|
+
content=find_packages_wrapper.on_failure_fn(),
|
|
47
|
+
additional_kwargs={"output": output},
|
|
48
|
+
status="error",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
return Command(
|
|
52
|
+
update=state.sanitize_update(
|
|
53
|
+
ctx=ctx,
|
|
54
|
+
update={
|
|
55
|
+
"agents_thoughts": [agent_thought, tool_message.content],
|
|
56
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
57
|
+
},
|
|
58
|
+
agent="executor",
|
|
59
|
+
),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
return find_packages
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
find_packages_wrapper = ToolWrapper(
|
|
66
|
+
tool_fn_getter=get_find_packages_tool,
|
|
67
|
+
on_success_fn=lambda thought, output: f"Packages found successfully ({thought}): {output}",
|
|
68
|
+
on_failure_fn=lambda: "Failed to find packages.",
|
|
69
|
+
)
|
|
@@ -1,17 +1,43 @@
|
|
|
1
|
-
from
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
2
4
|
|
|
3
5
|
from langchain_core.messages import ToolMessage
|
|
4
6
|
from langchain_core.tools import tool
|
|
5
7
|
from langchain_core.tools.base import InjectedToolCallId
|
|
8
|
+
from langgraph.prebuilt import InjectedState
|
|
6
9
|
from langgraph.types import Command
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
from typing import Annotated
|
|
12
|
+
|
|
13
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
14
|
+
from minitap.mobile_use.context import MobileUseContext
|
|
7
15
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
8
16
|
input_text as input_text_controller,
|
|
9
17
|
)
|
|
10
|
-
from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
|
|
11
|
-
from typing_extensions import Annotated
|
|
12
18
|
from minitap.mobile_use.graph.state import State
|
|
13
|
-
from
|
|
14
|
-
from minitap.mobile_use.
|
|
19
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
20
|
+
from minitap.mobile_use.tools.utils import focus_element_if_needed, move_cursor_to_end_if_bounds
|
|
21
|
+
from minitap.mobile_use.utils.logger import get_logger
|
|
22
|
+
|
|
23
|
+
logger = get_logger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class InputResult(BaseModel):
|
|
27
|
+
"""Result of an input operation from the controller layer."""
|
|
28
|
+
|
|
29
|
+
ok: bool
|
|
30
|
+
error: str | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _controller_input_text(ctx: MobileUseContext, text: str) -> InputResult:
|
|
34
|
+
"""
|
|
35
|
+
Thin wrapper to normalize the controller result.
|
|
36
|
+
"""
|
|
37
|
+
controller_out = input_text_controller(ctx=ctx, text=text)
|
|
38
|
+
if controller_out is None:
|
|
39
|
+
return InputResult(ok=True)
|
|
40
|
+
return InputResult(ok=False, error=str(controller_out))
|
|
15
41
|
|
|
16
42
|
|
|
17
43
|
def get_input_text_tool(ctx: MobileUseContext):
|
|
@@ -20,47 +46,44 @@ def get_input_text_tool(ctx: MobileUseContext):
|
|
|
20
46
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
47
|
state: Annotated[State, InjectedState],
|
|
22
48
|
agent_thought: str,
|
|
23
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
24
49
|
text: str,
|
|
50
|
+
text_input_resource_id: str,
|
|
25
51
|
):
|
|
26
52
|
"""
|
|
27
|
-
|
|
53
|
+
Focus a text field and type text into it.
|
|
28
54
|
|
|
29
|
-
|
|
30
|
-
|
|
55
|
+
- Ensure the corresponding element is focused (tap if necessary).
|
|
56
|
+
- If bounds are available, tap near the end to place the cursor at the end.
|
|
57
|
+
- Type the provided `text` using the controller.
|
|
58
|
+
"""
|
|
59
|
+
focused = focus_element_if_needed(ctx=ctx, resource_id=text_input_resource_id)
|
|
60
|
+
if focused:
|
|
61
|
+
move_cursor_to_end_if_bounds(ctx=ctx, state=state, resource_id=text_input_resource_id)
|
|
31
62
|
|
|
32
|
-
|
|
33
|
-
- Unicode not supported on Android.
|
|
63
|
+
result = _controller_input_text(ctx=ctx, text=text)
|
|
34
64
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
65
|
+
status: Literal["success", "error"] = "success" if result.ok else "error"
|
|
66
|
+
content_msg = (
|
|
67
|
+
input_text_wrapper.on_success_fn(text)
|
|
68
|
+
if result.ok
|
|
69
|
+
else input_text_wrapper.on_failure_fn(text)
|
|
70
|
+
)
|
|
40
71
|
|
|
41
|
-
Tip:
|
|
42
|
-
Use `copyTextFrom` to reuse generated inputs in later steps.
|
|
43
|
-
"""
|
|
44
|
-
output = input_text_controller(ctx=ctx, text=text)
|
|
45
|
-
has_failed = output is not None
|
|
46
72
|
tool_message = ToolMessage(
|
|
47
73
|
tool_call_id=tool_call_id,
|
|
48
|
-
content=
|
|
49
|
-
if
|
|
50
|
-
|
|
51
|
-
additional_kwargs={"error": output} if has_failed else {},
|
|
74
|
+
content=content_msg,
|
|
75
|
+
additional_kwargs={"error": result.error} if not result.ok else {},
|
|
76
|
+
status=status,
|
|
52
77
|
)
|
|
78
|
+
|
|
53
79
|
return Command(
|
|
54
|
-
update=
|
|
80
|
+
update=state.sanitize_update(
|
|
55
81
|
ctx=ctx,
|
|
56
|
-
|
|
57
|
-
executor_metadata=executor_metadata,
|
|
58
|
-
tool_message=tool_message,
|
|
59
|
-
is_failure=has_failed,
|
|
60
|
-
updates={
|
|
82
|
+
update={
|
|
61
83
|
"agents_thoughts": [agent_thought],
|
|
62
|
-
|
|
84
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
63
85
|
},
|
|
86
|
+
agent="executor",
|
|
64
87
|
),
|
|
65
88
|
)
|
|
66
89
|
|
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.types import Command
|
|
5
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
7
6
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
8
7
|
launch_app as launch_app_controller,
|
|
9
8
|
)
|
|
10
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
11
|
-
from
|
|
9
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
10
|
+
from typing import Annotated
|
|
12
11
|
from minitap.mobile_use.context import MobileUseContext
|
|
13
12
|
from minitap.mobile_use.graph.state import State
|
|
14
13
|
from langgraph.prebuilt import InjectedState
|
|
@@ -20,7 +19,6 @@ def get_launch_app_tool(ctx: MobileUseContext):
|
|
|
20
19
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
20
|
state: Annotated[State, InjectedState],
|
|
22
21
|
agent_thought: str,
|
|
23
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
24
22
|
package_name: str,
|
|
25
23
|
):
|
|
26
24
|
"""
|
|
@@ -34,18 +32,16 @@ def get_launch_app_tool(ctx: MobileUseContext):
|
|
|
34
32
|
if has_failed
|
|
35
33
|
else launch_app_wrapper.on_success_fn(package_name),
|
|
36
34
|
additional_kwargs={"error": output} if has_failed else {},
|
|
35
|
+
status="error" if has_failed else "success",
|
|
37
36
|
)
|
|
38
37
|
return Command(
|
|
39
|
-
update=
|
|
38
|
+
update=state.sanitize_update(
|
|
40
39
|
ctx=ctx,
|
|
41
|
-
|
|
42
|
-
executor_metadata=executor_metadata,
|
|
43
|
-
tool_message=tool_message,
|
|
44
|
-
is_failure=has_failed,
|
|
45
|
-
updates={
|
|
40
|
+
update={
|
|
46
41
|
"agents_thoughts": [agent_thought],
|
|
47
|
-
|
|
42
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
48
43
|
},
|
|
44
|
+
agent="executor",
|
|
49
45
|
),
|
|
50
46
|
)
|
|
51
47
|
|
|
@@ -1,18 +1,17 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.prebuilt import InjectedState
|
|
7
5
|
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
7
|
from minitap.mobile_use.context import MobileUseContext
|
|
9
8
|
from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
|
|
10
9
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
11
10
|
long_press_on as long_press_on_controller,
|
|
12
11
|
)
|
|
13
12
|
from minitap.mobile_use.graph.state import State
|
|
14
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
15
|
-
from
|
|
13
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
14
|
+
from typing import Annotated
|
|
16
15
|
|
|
17
16
|
|
|
18
17
|
def get_long_press_on_tool(ctx: MobileUseContext):
|
|
@@ -21,9 +20,8 @@ def get_long_press_on_tool(ctx: MobileUseContext):
|
|
|
21
20
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
22
21
|
state: Annotated[State, InjectedState],
|
|
23
22
|
agent_thought: str,
|
|
24
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
25
23
|
selector_request: SelectorRequest,
|
|
26
|
-
index:
|
|
24
|
+
index: int | None = None,
|
|
27
25
|
):
|
|
28
26
|
"""
|
|
29
27
|
Long press on a UI element identified by the given selector.
|
|
@@ -37,18 +35,16 @@ def get_long_press_on_tool(ctx: MobileUseContext):
|
|
|
37
35
|
if has_failed
|
|
38
36
|
else long_press_on_wrapper.on_success_fn(),
|
|
39
37
|
additional_kwargs={"error": output} if has_failed else {},
|
|
38
|
+
status="error" if has_failed else "success",
|
|
40
39
|
)
|
|
41
40
|
return Command(
|
|
42
|
-
update=
|
|
41
|
+
update=state.sanitize_update(
|
|
43
42
|
ctx=ctx,
|
|
44
|
-
|
|
45
|
-
executor_metadata=executor_metadata,
|
|
46
|
-
tool_message=tool_message,
|
|
47
|
-
is_failure=has_failed,
|
|
48
|
-
updates={
|
|
43
|
+
update={
|
|
49
44
|
"agents_thoughts": [agent_thought],
|
|
50
|
-
|
|
45
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
51
46
|
},
|
|
47
|
+
agent="executor",
|
|
52
48
|
),
|
|
53
49
|
)
|
|
54
50
|
|
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.prebuilt import InjectedState
|
|
7
5
|
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
7
|
from minitap.mobile_use.context import MobileUseContext
|
|
9
8
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
10
9
|
open_link as open_link_controller,
|
|
11
10
|
)
|
|
12
11
|
from minitap.mobile_use.graph.state import State
|
|
13
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
14
|
-
from
|
|
12
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
13
|
+
from typing import Annotated
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
def get_open_link_tool(ctx: MobileUseContext):
|
|
@@ -20,7 +19,6 @@ def get_open_link_tool(ctx: MobileUseContext):
|
|
|
20
19
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
20
|
state: Annotated[State, InjectedState],
|
|
22
21
|
agent_thought: str,
|
|
23
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
24
22
|
url: str,
|
|
25
23
|
):
|
|
26
24
|
"""
|
|
@@ -34,18 +32,16 @@ def get_open_link_tool(ctx: MobileUseContext):
|
|
|
34
32
|
if has_failed
|
|
35
33
|
else open_link_wrapper.on_success_fn(url),
|
|
36
34
|
additional_kwargs={"error": output} if has_failed else {},
|
|
35
|
+
status="error" if has_failed else "success",
|
|
37
36
|
)
|
|
38
37
|
return Command(
|
|
39
|
-
update=
|
|
38
|
+
update=state.sanitize_update(
|
|
40
39
|
ctx=ctx,
|
|
41
|
-
|
|
42
|
-
executor_metadata=executor_metadata,
|
|
43
|
-
tool_message=tool_message,
|
|
44
|
-
is_failure=has_failed,
|
|
45
|
-
updates={
|
|
40
|
+
update={
|
|
46
41
|
"agents_thoughts": [agent_thought],
|
|
47
|
-
|
|
42
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
48
43
|
},
|
|
44
|
+
agent="executor",
|
|
49
45
|
),
|
|
50
46
|
)
|
|
51
47
|
|
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.types import Command
|
|
5
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
7
6
|
from minitap.mobile_use.context import MobileUseContext
|
|
8
7
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
9
8
|
paste_text as paste_text_controller,
|
|
10
9
|
)
|
|
11
10
|
from minitap.mobile_use.graph.state import State
|
|
12
11
|
from langgraph.prebuilt import InjectedState
|
|
13
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
14
|
-
from
|
|
12
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
13
|
+
from typing import Annotated
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
def get_paste_text_tool(ctx: MobileUseContext):
|
|
@@ -20,7 +19,6 @@ def get_paste_text_tool(ctx: MobileUseContext):
|
|
|
20
19
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
21
20
|
state: Annotated[State, InjectedState],
|
|
22
21
|
agent_thought: str,
|
|
23
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
24
22
|
):
|
|
25
23
|
"""
|
|
26
24
|
Pastes text previously copied via `copyTextFrom` into the currently focused field.
|
|
@@ -41,18 +39,16 @@ def get_paste_text_tool(ctx: MobileUseContext):
|
|
|
41
39
|
if has_failed
|
|
42
40
|
else paste_text_wrapper.on_success_fn(),
|
|
43
41
|
additional_kwargs={"error": output} if has_failed else {},
|
|
42
|
+
status="error" if has_failed else "success",
|
|
44
43
|
)
|
|
45
44
|
return Command(
|
|
46
|
-
update=
|
|
45
|
+
update=state.sanitize_update(
|
|
47
46
|
ctx=ctx,
|
|
48
|
-
|
|
49
|
-
executor_metadata=executor_metadata,
|
|
50
|
-
tool_message=tool_message,
|
|
51
|
-
is_failure=has_failed,
|
|
52
|
-
updates={
|
|
47
|
+
update={
|
|
53
48
|
"agents_thoughts": [agent_thought],
|
|
54
|
-
|
|
49
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
55
50
|
},
|
|
51
|
+
agent="executor",
|
|
56
52
|
),
|
|
57
53
|
)
|
|
58
54
|
|
|
@@ -1,18 +1,17 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.prebuilt import InjectedState
|
|
7
5
|
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
7
|
from minitap.mobile_use.context import MobileUseContext
|
|
9
8
|
from minitap.mobile_use.controllers.mobile_command_controller import Key
|
|
10
9
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
11
10
|
press_key as press_key_controller,
|
|
12
11
|
)
|
|
13
12
|
from minitap.mobile_use.graph.state import State
|
|
14
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
15
|
-
from
|
|
13
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
14
|
+
from typing import Annotated
|
|
16
15
|
|
|
17
16
|
|
|
18
17
|
def get_press_key_tool(ctx: MobileUseContext):
|
|
@@ -21,7 +20,6 @@ def get_press_key_tool(ctx: MobileUseContext):
|
|
|
21
20
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
22
21
|
state: Annotated[State, InjectedState],
|
|
23
22
|
agent_thought: str,
|
|
24
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
25
23
|
key: Key,
|
|
26
24
|
):
|
|
27
25
|
"""Press a key on the device."""
|
|
@@ -33,18 +31,16 @@ def get_press_key_tool(ctx: MobileUseContext):
|
|
|
33
31
|
if has_failed
|
|
34
32
|
else press_key_wrapper.on_success_fn(key),
|
|
35
33
|
additional_kwargs={"error": output} if has_failed else {},
|
|
34
|
+
status="error" if has_failed else "success",
|
|
36
35
|
)
|
|
37
36
|
return Command(
|
|
38
|
-
update=
|
|
37
|
+
update=state.sanitize_update(
|
|
39
38
|
ctx=ctx,
|
|
40
|
-
|
|
41
|
-
executor_metadata=executor_metadata,
|
|
42
|
-
tool_message=tool_message,
|
|
43
|
-
is_failure=has_failed,
|
|
44
|
-
updates={
|
|
39
|
+
update={
|
|
45
40
|
"agents_thoughts": [agent_thought],
|
|
46
|
-
|
|
41
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
47
42
|
},
|
|
43
|
+
agent="executor",
|
|
48
44
|
),
|
|
49
45
|
)
|
|
50
46
|
|
|
@@ -1,15 +1,14 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.prebuilt import InjectedState
|
|
7
5
|
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
7
|
from minitap.mobile_use.context import MobileUseContext
|
|
9
8
|
from minitap.mobile_use.controllers.mobile_command_controller import stop_app as stop_app_controller
|
|
10
9
|
from minitap.mobile_use.graph.state import State
|
|
11
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
12
|
-
from
|
|
10
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
11
|
+
from typing import Annotated
|
|
13
12
|
|
|
14
13
|
|
|
15
14
|
def get_stop_app_tool(ctx: MobileUseContext):
|
|
@@ -18,8 +17,7 @@ def get_stop_app_tool(ctx: MobileUseContext):
|
|
|
18
17
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
19
18
|
state: Annotated[State, InjectedState],
|
|
20
19
|
agent_thought: str,
|
|
21
|
-
|
|
22
|
-
package_name: Optional[str] = None,
|
|
20
|
+
package_name: str | None = None,
|
|
23
21
|
):
|
|
24
22
|
"""
|
|
25
23
|
Stops current application if it is running.
|
|
@@ -33,18 +31,16 @@ def get_stop_app_tool(ctx: MobileUseContext):
|
|
|
33
31
|
if has_failed
|
|
34
32
|
else stop_app_wrapper.on_success_fn(package_name),
|
|
35
33
|
additional_kwargs={"error": output} if has_failed else {},
|
|
34
|
+
status="error" if has_failed else "success",
|
|
36
35
|
)
|
|
37
36
|
return Command(
|
|
38
|
-
update=
|
|
37
|
+
update=state.sanitize_update(
|
|
39
38
|
ctx=ctx,
|
|
40
|
-
|
|
41
|
-
executor_metadata=executor_metadata,
|
|
42
|
-
tool_message=tool_message,
|
|
43
|
-
is_failure=has_failed,
|
|
44
|
-
updates={
|
|
39
|
+
update={
|
|
45
40
|
"agents_thoughts": [agent_thought],
|
|
46
|
-
|
|
41
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
47
42
|
},
|
|
43
|
+
agent="executor",
|
|
48
44
|
),
|
|
49
45
|
)
|
|
50
46
|
|
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
|
-
|
|
3
1
|
from langchain_core.messages import ToolMessage
|
|
4
2
|
from langchain_core.tools import tool
|
|
5
3
|
from langchain_core.tools.base import InjectedToolCallId
|
|
6
4
|
from langgraph.prebuilt import InjectedState
|
|
7
5
|
from langgraph.types import Command
|
|
6
|
+
from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
|
|
8
7
|
from minitap.mobile_use.context import MobileUseContext
|
|
9
8
|
from minitap.mobile_use.controllers.mobile_command_controller import SwipeRequest
|
|
10
9
|
from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
|
|
11
10
|
from minitap.mobile_use.graph.state import State
|
|
12
|
-
from minitap.mobile_use.tools.tool_wrapper import
|
|
13
|
-
from
|
|
11
|
+
from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
|
|
12
|
+
from typing import Annotated
|
|
14
13
|
|
|
15
14
|
|
|
16
15
|
def get_swipe_tool(ctx: MobileUseContext):
|
|
@@ -19,7 +18,6 @@ def get_swipe_tool(ctx: MobileUseContext):
|
|
|
19
18
|
tool_call_id: Annotated[str, InjectedToolCallId],
|
|
20
19
|
state: Annotated[State, InjectedState],
|
|
21
20
|
agent_thought: str,
|
|
22
|
-
executor_metadata: Optional[ExecutorMetadata],
|
|
23
21
|
swipe_request: SwipeRequest,
|
|
24
22
|
):
|
|
25
23
|
"""
|
|
@@ -31,18 +29,16 @@ def get_swipe_tool(ctx: MobileUseContext):
|
|
|
31
29
|
tool_call_id=tool_call_id,
|
|
32
30
|
content=swipe_wrapper.on_failure_fn() if has_failed else swipe_wrapper.on_success_fn(),
|
|
33
31
|
additional_kwargs={"error": output} if has_failed else {},
|
|
32
|
+
status="error" if has_failed else "success",
|
|
34
33
|
)
|
|
35
34
|
return Command(
|
|
36
|
-
update=
|
|
35
|
+
update=state.sanitize_update(
|
|
37
36
|
ctx=ctx,
|
|
38
|
-
|
|
39
|
-
executor_metadata=executor_metadata,
|
|
40
|
-
tool_message=tool_message,
|
|
41
|
-
is_failure=has_failed,
|
|
42
|
-
updates={
|
|
37
|
+
update={
|
|
43
38
|
"agents_thoughts": [agent_thought],
|
|
44
|
-
|
|
39
|
+
EXECUTOR_MESSAGES_KEY: [tool_message],
|
|
45
40
|
},
|
|
41
|
+
agent="executor",
|
|
46
42
|
),
|
|
47
43
|
)
|
|
48
44
|
|