minitap-mobile-use 2.0.0__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of minitap-mobile-use might be problematic. Click here for more details.
- minitap/mobile_use/agents/cortex/cortex.md +19 -10
- minitap/mobile_use/agents/cortex/cortex.py +15 -2
- minitap/mobile_use/agents/cortex/types.py +2 -4
- minitap/mobile_use/agents/executor/executor.md +20 -15
- minitap/mobile_use/agents/executor/executor.py +6 -18
- minitap/mobile_use/agents/executor/tool_node.py +105 -0
- minitap/mobile_use/agents/hopper/hopper.md +2 -10
- minitap/mobile_use/agents/hopper/hopper.py +4 -9
- minitap/mobile_use/agents/orchestrator/human.md +3 -4
- minitap/mobile_use/agents/orchestrator/orchestrator.md +25 -7
- minitap/mobile_use/agents/orchestrator/orchestrator.py +56 -56
- minitap/mobile_use/agents/orchestrator/types.py +5 -8
- minitap/mobile_use/agents/outputter/outputter.py +1 -2
- minitap/mobile_use/agents/planner/planner.md +25 -15
- minitap/mobile_use/agents/planner/planner.py +7 -1
- minitap/mobile_use/agents/planner/types.py +10 -5
- minitap/mobile_use/agents/planner/utils.py +11 -0
- minitap/mobile_use/agents/summarizer/summarizer.py +2 -1
- minitap/mobile_use/clients/device_hardware_client.py +3 -0
- minitap/mobile_use/config.py +16 -14
- minitap/mobile_use/constants.py +1 -0
- minitap/mobile_use/context.py +3 -4
- minitap/mobile_use/controllers/mobile_command_controller.py +37 -26
- minitap/mobile_use/controllers/platform_specific_commands_controller.py +3 -4
- minitap/mobile_use/graph/graph.py +10 -31
- minitap/mobile_use/graph/state.py +34 -14
- minitap/mobile_use/main.py +11 -8
- minitap/mobile_use/sdk/agent.py +78 -63
- minitap/mobile_use/sdk/builders/agent_config_builder.py +23 -11
- minitap/mobile_use/sdk/builders/task_request_builder.py +9 -9
- minitap/mobile_use/sdk/examples/smart_notification_assistant.py +1 -2
- minitap/mobile_use/sdk/types/agent.py +10 -5
- minitap/mobile_use/sdk/types/task.py +19 -18
- minitap/mobile_use/sdk/utils.py +1 -1
- minitap/mobile_use/servers/config.py +1 -2
- minitap/mobile_use/servers/device_hardware_bridge.py +3 -4
- minitap/mobile_use/servers/start_servers.py +4 -4
- minitap/mobile_use/servers/stop_servers.py +12 -18
- minitap/mobile_use/services/llm.py +4 -2
- minitap/mobile_use/tools/index.py +11 -7
- minitap/mobile_use/tools/mobile/back.py +8 -12
- minitap/mobile_use/tools/mobile/clear_text.py +277 -0
- minitap/mobile_use/tools/mobile/copy_text_from.py +8 -12
- minitap/mobile_use/tools/mobile/erase_one_char.py +56 -0
- minitap/mobile_use/tools/mobile/find_packages.py +69 -0
- minitap/mobile_use/tools/mobile/input_text.py +55 -32
- minitap/mobile_use/tools/mobile/launch_app.py +8 -12
- minitap/mobile_use/tools/mobile/long_press_on.py +9 -13
- minitap/mobile_use/tools/mobile/open_link.py +8 -12
- minitap/mobile_use/tools/mobile/paste_text.py +8 -12
- minitap/mobile_use/tools/mobile/press_key.py +8 -12
- minitap/mobile_use/tools/mobile/stop_app.py +9 -13
- minitap/mobile_use/tools/mobile/swipe.py +8 -12
- minitap/mobile_use/tools/mobile/take_screenshot.py +8 -12
- minitap/mobile_use/tools/mobile/tap.py +9 -13
- minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +9 -13
- minitap/mobile_use/tools/tool_wrapper.py +1 -23
- minitap/mobile_use/tools/utils.py +86 -0
- minitap/mobile_use/utils/cli_helpers.py +1 -2
- minitap/mobile_use/utils/cli_selection.py +5 -6
- minitap/mobile_use/utils/decorators.py +21 -20
- minitap/mobile_use/utils/logger.py +3 -4
- minitap/mobile_use/utils/media.py +1 -1
- minitap/mobile_use/utils/recorder.py +11 -10
- minitap/mobile_use/utils/ui_hierarchy.py +98 -3
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/METADATA +12 -2
- minitap_mobile_use-2.1.0.dist-info/RECORD +96 -0
- minitap/mobile_use/agents/executor/executor_context_cleaner.py +0 -27
- minitap/mobile_use/tools/mobile/erase_text.py +0 -124
- minitap/mobile_use/tools/mobile/list_packages.py +0 -78
- minitap/mobile_use/tools/mobile/run_flow.py +0 -57
- minitap_mobile_use-2.0.0.dist-info/RECORD +0 -95
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/WHEEL +0 -0
- {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.1.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from langchain_core.messages import AIMessage, AnyMessage
|
|
2
2
|
from langgraph.graph import add_messages
|
|
3
3
|
from langgraph.prebuilt.chat_agent_executor import AgentStatePydantic
|
|
4
|
-
from
|
|
4
|
+
from typing import Annotated
|
|
5
5
|
|
|
6
6
|
from minitap.mobile_use.agents.planner.types import Subgoal
|
|
7
|
+
from minitap.mobile_use.config import AgentNode
|
|
7
8
|
from minitap.mobile_use.utils.logger import get_logger
|
|
8
9
|
from minitap.mobile_use.utils.recorder import record_interaction
|
|
9
10
|
from minitap.mobile_use.context import MobileUseContext
|
|
@@ -23,51 +24,70 @@ class State(AgentStatePydantic):
|
|
|
23
24
|
subgoal_plan: Annotated[list[Subgoal], "The current plan, made of subgoals"]
|
|
24
25
|
|
|
25
26
|
# contextor related keys
|
|
26
|
-
latest_screenshot_base64: Annotated[
|
|
27
|
+
latest_screenshot_base64: Annotated[str | None, "Latest screenshot of the device", take_last]
|
|
27
28
|
latest_ui_hierarchy: Annotated[
|
|
28
|
-
|
|
29
|
+
list[dict] | None, "Latest UI hierarchy of the device", take_last
|
|
29
30
|
]
|
|
30
|
-
focused_app_info: Annotated[
|
|
31
|
-
device_date: Annotated[
|
|
31
|
+
focused_app_info: Annotated[str | None, "Focused app info", take_last]
|
|
32
|
+
device_date: Annotated[str | None, "Date of the device", take_last]
|
|
32
33
|
|
|
33
34
|
# cortex related keys
|
|
34
35
|
structured_decisions: Annotated[
|
|
35
|
-
|
|
36
|
+
str | None,
|
|
36
37
|
"Structured decisions made by the cortex, for the executor to follow",
|
|
37
38
|
take_last,
|
|
38
39
|
]
|
|
40
|
+
complete_subgoals_by_ids: Annotated[
|
|
41
|
+
list[str],
|
|
42
|
+
"List of subgoal IDs to complete",
|
|
43
|
+
take_last,
|
|
44
|
+
]
|
|
39
45
|
|
|
40
46
|
# executor related keys
|
|
41
|
-
executor_retrigger: Annotated[Optional[bool], "Whether the executor must be retriggered"]
|
|
42
|
-
executor_failed: Annotated[bool, "Whether a tool call made by the executor failed"]
|
|
43
47
|
executor_messages: Annotated[list[AnyMessage], "Sequential Executor messages", add_messages]
|
|
44
|
-
cortex_last_thought: Annotated[
|
|
48
|
+
cortex_last_thought: Annotated[str | None, "Last thought of the cortex for the executor"]
|
|
45
49
|
|
|
46
50
|
# common keys
|
|
47
51
|
agents_thoughts: Annotated[
|
|
48
52
|
list[str],
|
|
49
53
|
"All thoughts and reasons that led to actions (why a tool was called, expected outcomes..)",
|
|
54
|
+
take_last,
|
|
50
55
|
]
|
|
51
56
|
|
|
52
|
-
def sanitize_update(
|
|
57
|
+
def sanitize_update(
|
|
58
|
+
self,
|
|
59
|
+
ctx: MobileUseContext,
|
|
60
|
+
update: dict,
|
|
61
|
+
agent: AgentNode | None = None,
|
|
62
|
+
):
|
|
53
63
|
"""
|
|
54
64
|
Sanitizes the state update to ensure it is valid and apply side effect logic where required.
|
|
65
|
+
The agent is required if the update contains the "agents_thoughts" key.
|
|
55
66
|
"""
|
|
56
|
-
updated_agents_thoughts:
|
|
67
|
+
updated_agents_thoughts: str | list[str] | None = update.get("agents_thoughts", None)
|
|
57
68
|
if updated_agents_thoughts is not None:
|
|
58
69
|
if isinstance(updated_agents_thoughts, str):
|
|
59
70
|
updated_agents_thoughts = [updated_agents_thoughts]
|
|
60
71
|
elif not isinstance(updated_agents_thoughts, list):
|
|
61
72
|
raise ValueError("agents_thoughts must be a str or list[str]")
|
|
73
|
+
if agent is None:
|
|
74
|
+
raise ValueError("Agent is required when updating the 'agents_thoughts' key")
|
|
62
75
|
update["agents_thoughts"] = _add_agent_thoughts(
|
|
63
76
|
ctx=ctx,
|
|
64
77
|
old=self.agents_thoughts,
|
|
65
78
|
new=updated_agents_thoughts,
|
|
79
|
+
agent=agent,
|
|
66
80
|
)
|
|
67
81
|
return update
|
|
68
82
|
|
|
69
83
|
|
|
70
|
-
def _add_agent_thoughts(
|
|
84
|
+
def _add_agent_thoughts(
|
|
85
|
+
ctx: MobileUseContext,
|
|
86
|
+
old: list[str],
|
|
87
|
+
new: list[str],
|
|
88
|
+
agent: AgentNode,
|
|
89
|
+
) -> list[str]:
|
|
90
|
+
named_thoughts = [f"[{agent}] {thought}" for thought in new]
|
|
71
91
|
if ctx.execution_setup:
|
|
72
|
-
record_interaction(ctx, response=AIMessage(content=str(
|
|
73
|
-
return old +
|
|
92
|
+
record_interaction(ctx, response=AIMessage(content=str(named_thoughts)))
|
|
93
|
+
return old + named_thoughts
|
minitap/mobile_use/main.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
|
-
from adbutils import AdbClient
|
|
4
|
-
from typing import Optional
|
|
5
3
|
|
|
6
4
|
import typer
|
|
5
|
+
from adbutils import AdbClient
|
|
6
|
+
from langchain.callbacks.base import Callbacks
|
|
7
7
|
from rich.console import Console
|
|
8
|
-
from
|
|
8
|
+
from typing import Annotated
|
|
9
9
|
|
|
10
10
|
from minitap.mobile_use.config import (
|
|
11
11
|
initialize_llm_config,
|
|
@@ -23,9 +23,10 @@ logger = get_logger(__name__)
|
|
|
23
23
|
|
|
24
24
|
async def run_automation(
|
|
25
25
|
goal: str,
|
|
26
|
-
test_name:
|
|
26
|
+
test_name: str | None = None,
|
|
27
27
|
traces_output_path_str: str = "traces",
|
|
28
|
-
output_description:
|
|
28
|
+
output_description: str | None = None,
|
|
29
|
+
graph_config_callbacks: Callbacks = [],
|
|
29
30
|
):
|
|
30
31
|
llm_config = initialize_llm_config()
|
|
31
32
|
agent_profile = AgentProfile(name="default", llm_config=llm_config)
|
|
@@ -37,11 +38,13 @@ async def run_automation(
|
|
|
37
38
|
config.with_hw_bridge_base_url(url=settings.DEVICE_HARDWARE_BRIDGE_BASE_URL)
|
|
38
39
|
if settings.DEVICE_SCREEN_API_BASE_URL:
|
|
39
40
|
config.with_screen_api_base_url(url=settings.DEVICE_SCREEN_API_BASE_URL)
|
|
41
|
+
if graph_config_callbacks:
|
|
42
|
+
config.with_graph_config_callbacks(graph_config_callbacks)
|
|
40
43
|
|
|
41
44
|
agent = Agent(config=config.build())
|
|
42
45
|
agent.init(
|
|
43
46
|
retry_count=int(os.getenv("MOBILE_USE_HEALTH_RETRIES", 5)),
|
|
44
|
-
retry_wait_seconds=int(os.getenv("MOBILE_USE_HEALTH_DELAY",
|
|
47
|
+
retry_wait_seconds=int(os.getenv("MOBILE_USE_HEALTH_DELAY", 2)),
|
|
45
48
|
)
|
|
46
49
|
|
|
47
50
|
task = agent.new_task(goal)
|
|
@@ -66,7 +69,7 @@ async def run_automation(
|
|
|
66
69
|
def main(
|
|
67
70
|
goal: Annotated[str, typer.Argument(help="The main goal for the agent to achieve.")],
|
|
68
71
|
test_name: Annotated[
|
|
69
|
-
|
|
72
|
+
str | None,
|
|
70
73
|
typer.Option(
|
|
71
74
|
"--test-name",
|
|
72
75
|
"-n",
|
|
@@ -82,7 +85,7 @@ def main(
|
|
|
82
85
|
),
|
|
83
86
|
] = "traces",
|
|
84
87
|
output_description: Annotated[
|
|
85
|
-
|
|
88
|
+
str | None,
|
|
86
89
|
typer.Option(
|
|
87
90
|
"--output-description",
|
|
88
91
|
"-o",
|
minitap/mobile_use/sdk/agent.py
CHANGED
|
@@ -1,68 +1,67 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from datetime import datetime
|
|
3
|
-
from pathlib import Path
|
|
4
2
|
import sys
|
|
5
3
|
import tempfile
|
|
6
4
|
import time
|
|
7
|
-
from types import NoneType
|
|
8
|
-
from typing import Optional, TypeVar, overload
|
|
9
5
|
import uuid
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from types import NoneType
|
|
9
|
+
from typing import TypeVar, overload
|
|
10
|
+
|
|
10
11
|
from adbutils import AdbClient
|
|
11
12
|
from langchain_core.messages import AIMessage
|
|
12
13
|
from pydantic import BaseModel
|
|
13
|
-
from minitap.mobile_use.agents.outputter.outputter import outputter
|
|
14
14
|
|
|
15
|
+
from minitap.mobile_use.agents.outputter.outputter import outputter
|
|
16
|
+
from minitap.mobile_use.clients.device_hardware_client import DeviceHardwareClient
|
|
17
|
+
from minitap.mobile_use.clients.screen_api_client import ScreenApiClient
|
|
15
18
|
from minitap.mobile_use.config import OutputConfig, record_events
|
|
16
|
-
from minitap.mobile_use.graph.graph import get_graph
|
|
17
|
-
from minitap.mobile_use.graph.state import State
|
|
18
|
-
from minitap.mobile_use.sdk.builders.agent_config_builder import get_default_agent_config
|
|
19
|
-
from minitap.mobile_use.sdk.builders.task_request_builder import TaskRequestBuilder
|
|
20
|
-
from minitap.mobile_use.sdk.constants import (
|
|
21
|
-
DEFAULT_HW_BRIDGE_BASE_URL,
|
|
22
|
-
DEFAULT_SCREEN_API_BASE_URL,
|
|
23
|
-
)
|
|
24
|
-
from minitap.mobile_use.sdk.types.agent import AgentConfig
|
|
25
19
|
from minitap.mobile_use.context import (
|
|
26
20
|
DeviceContext,
|
|
27
21
|
DevicePlatform,
|
|
28
22
|
ExecutionSetup,
|
|
29
23
|
MobileUseContext,
|
|
30
24
|
)
|
|
31
|
-
from minitap.mobile_use.clients.device_hardware_client import DeviceHardwareClient
|
|
32
|
-
from minitap.mobile_use.clients.screen_api_client import ScreenApiClient
|
|
33
25
|
from minitap.mobile_use.controllers.mobile_command_controller import (
|
|
34
26
|
ScreenDataResponse,
|
|
35
27
|
get_screen_data,
|
|
36
28
|
)
|
|
37
29
|
from minitap.mobile_use.controllers.platform_specific_commands_controller import get_first_device
|
|
38
|
-
|
|
39
|
-
from minitap.mobile_use.
|
|
40
|
-
from minitap.mobile_use.
|
|
41
|
-
from minitap.mobile_use.
|
|
42
|
-
|
|
43
|
-
|
|
30
|
+
from minitap.mobile_use.graph.graph import get_graph
|
|
31
|
+
from minitap.mobile_use.graph.state import State
|
|
32
|
+
from minitap.mobile_use.sdk.builders.agent_config_builder import get_default_agent_config
|
|
33
|
+
from minitap.mobile_use.sdk.builders.task_request_builder import TaskRequestBuilder
|
|
34
|
+
from minitap.mobile_use.sdk.constants import (
|
|
35
|
+
DEFAULT_HW_BRIDGE_BASE_URL,
|
|
36
|
+
DEFAULT_SCREEN_API_BASE_URL,
|
|
44
37
|
)
|
|
45
|
-
from minitap.mobile_use.
|
|
38
|
+
from minitap.mobile_use.sdk.types.agent import AgentConfig
|
|
46
39
|
from minitap.mobile_use.sdk.types.exceptions import (
|
|
40
|
+
AgentNotInitializedError,
|
|
47
41
|
AgentProfileNotFoundError,
|
|
48
42
|
AgentTaskRequestError,
|
|
49
43
|
DeviceNotFoundError,
|
|
50
44
|
ServerStartupError,
|
|
51
|
-
AgentNotInitializedError,
|
|
52
45
|
)
|
|
53
46
|
from minitap.mobile_use.sdk.types.task import AgentProfile, Task, TaskRequest, TaskStatus
|
|
47
|
+
from minitap.mobile_use.servers.device_hardware_bridge import BridgeStatus
|
|
48
|
+
from minitap.mobile_use.servers.start_servers import (
|
|
49
|
+
start_device_hardware_bridge,
|
|
50
|
+
start_device_screen_api,
|
|
51
|
+
)
|
|
52
|
+
from minitap.mobile_use.servers.stop_servers import stop_servers
|
|
53
|
+
from minitap.mobile_use.utils.logger import get_logger
|
|
54
54
|
from minitap.mobile_use.utils.media import (
|
|
55
55
|
create_gif_from_trace_folder,
|
|
56
56
|
create_steps_json_from_trace_folder,
|
|
57
57
|
remove_images_from_trace_folder,
|
|
58
58
|
remove_steps_json_from_trace_folder,
|
|
59
59
|
)
|
|
60
|
-
from minitap.mobile_use.utils.recorder import
|
|
61
|
-
|
|
60
|
+
from minitap.mobile_use.utils.recorder import log_agent_thought
|
|
62
61
|
|
|
63
62
|
logger = get_logger(__name__)
|
|
64
63
|
|
|
65
|
-
TOutput = TypeVar("TOutput", bound=
|
|
64
|
+
TOutput = TypeVar("TOutput", bound=BaseModel | None)
|
|
66
65
|
|
|
67
66
|
|
|
68
67
|
class Agent:
|
|
@@ -75,9 +74,9 @@ class Agent:
|
|
|
75
74
|
_device_context: DeviceContext
|
|
76
75
|
_screen_api_client: ScreenApiClient
|
|
77
76
|
_hw_bridge_client: DeviceHardwareClient
|
|
78
|
-
_adb_client:
|
|
77
|
+
_adb_client: AdbClient | None
|
|
79
78
|
|
|
80
|
-
def __init__(self, config:
|
|
79
|
+
def __init__(self, config: AgentConfig | None = None):
|
|
81
80
|
self._config = config or get_default_agent_config()
|
|
82
81
|
self._tasks = []
|
|
83
82
|
self._tmp_traces_dir = Path(tempfile.gettempdir()) / "mobile-use-traces"
|
|
@@ -127,7 +126,10 @@ class Agent:
|
|
|
127
126
|
f"Server start failed, attempting restart "
|
|
128
127
|
f"{restart_attempt}/{server_restart_attempts}"
|
|
129
128
|
)
|
|
130
|
-
|
|
129
|
+
stop_servers(
|
|
130
|
+
should_stop_screen_api=self._is_default_screen_api,
|
|
131
|
+
should_stop_hw_bridge=self._is_default_hw_bridge,
|
|
132
|
+
)
|
|
131
133
|
else:
|
|
132
134
|
error_msg = "Mobile-use servers failed to start after all restart attempts."
|
|
133
135
|
logger.error(error_msg)
|
|
@@ -151,9 +153,9 @@ class Agent:
|
|
|
151
153
|
*,
|
|
152
154
|
goal: str,
|
|
153
155
|
output: type[TOutput],
|
|
154
|
-
profile:
|
|
155
|
-
name:
|
|
156
|
-
) ->
|
|
156
|
+
profile: str | AgentProfile | None = None,
|
|
157
|
+
name: str | None = None,
|
|
158
|
+
) -> TOutput | None: ...
|
|
157
159
|
|
|
158
160
|
@overload
|
|
159
161
|
async def run_task(
|
|
@@ -161,9 +163,9 @@ class Agent:
|
|
|
161
163
|
*,
|
|
162
164
|
goal: str,
|
|
163
165
|
output: str,
|
|
164
|
-
profile:
|
|
165
|
-
name:
|
|
166
|
-
) ->
|
|
166
|
+
profile: str | AgentProfile | None = None,
|
|
167
|
+
name: str | None = None,
|
|
168
|
+
) -> str | dict | None: ...
|
|
167
169
|
|
|
168
170
|
@overload
|
|
169
171
|
async def run_task(
|
|
@@ -171,25 +173,25 @@ class Agent:
|
|
|
171
173
|
*,
|
|
172
174
|
goal: str,
|
|
173
175
|
output=None,
|
|
174
|
-
profile:
|
|
175
|
-
name:
|
|
176
|
-
) ->
|
|
176
|
+
profile: str | AgentProfile | None = None,
|
|
177
|
+
name: str | None = None,
|
|
178
|
+
) -> str | None: ...
|
|
177
179
|
|
|
178
180
|
@overload
|
|
179
|
-
async def run_task(self, *, request: TaskRequest[None]) ->
|
|
181
|
+
async def run_task(self, *, request: TaskRequest[None]) -> str | dict | None: ...
|
|
180
182
|
|
|
181
183
|
@overload
|
|
182
|
-
async def run_task(self, *, request: TaskRequest[TOutput]) ->
|
|
184
|
+
async def run_task(self, *, request: TaskRequest[TOutput]) -> TOutput | None: ...
|
|
183
185
|
|
|
184
186
|
async def run_task(
|
|
185
187
|
self,
|
|
186
188
|
*,
|
|
187
|
-
goal:
|
|
188
|
-
output:
|
|
189
|
-
profile:
|
|
190
|
-
name:
|
|
191
|
-
request:
|
|
192
|
-
) ->
|
|
189
|
+
goal: str | None = None,
|
|
190
|
+
output: type[TOutput] | str | None = None,
|
|
191
|
+
profile: str | AgentProfile | None = None,
|
|
192
|
+
name: str | None = None,
|
|
193
|
+
request: TaskRequest[TOutput] | None = None,
|
|
194
|
+
) -> str | dict | TOutput | None:
|
|
193
195
|
if request is not None:
|
|
194
196
|
return await self._run_task(request)
|
|
195
197
|
if goal is None:
|
|
@@ -206,7 +208,7 @@ class Agent:
|
|
|
206
208
|
task_request.with_name(name=name)
|
|
207
209
|
return await self._run_task(task_request.build())
|
|
208
210
|
|
|
209
|
-
async def _run_task(self, request: TaskRequest[TOutput]) ->
|
|
211
|
+
async def _run_task(self, request: TaskRequest[TOutput]) -> str | dict | TOutput | None:
|
|
210
212
|
if not self._initialized:
|
|
211
213
|
raise AgentNotInitializedError()
|
|
212
214
|
|
|
@@ -261,17 +263,31 @@ class Agent:
|
|
|
261
263
|
input=graph_input,
|
|
262
264
|
config={
|
|
263
265
|
"recursion_limit": task.request.max_steps,
|
|
266
|
+
"callbacks": self._config.graph_config_callbacks,
|
|
264
267
|
},
|
|
265
|
-
stream_mode=["messages", "custom", "values"],
|
|
268
|
+
stream_mode=["messages", "custom", "updates", "values"],
|
|
266
269
|
):
|
|
267
|
-
stream_mode,
|
|
270
|
+
stream_mode, payload = chunk
|
|
268
271
|
if stream_mode == "values":
|
|
269
|
-
last_state_snapshot =
|
|
272
|
+
last_state_snapshot = payload # type: ignore
|
|
270
273
|
last_state = State(**last_state_snapshot) # type: ignore
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
274
|
+
if task.request.thoughts_output_path:
|
|
275
|
+
record_events(
|
|
276
|
+
output_path=task.request.thoughts_output_path,
|
|
277
|
+
events=last_state.agents_thoughts,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
if stream_mode == "updates":
|
|
281
|
+
for key, value in payload.items(): # type: ignore
|
|
282
|
+
if value and "agents_thoughts" in value:
|
|
283
|
+
new_thoughts = value["agents_thoughts"]
|
|
284
|
+
last_item = new_thoughts[-1] if new_thoughts else None
|
|
285
|
+
if last_item:
|
|
286
|
+
log_agent_thought(
|
|
287
|
+
prefix=key,
|
|
288
|
+
agent_thought=last_item,
|
|
289
|
+
)
|
|
290
|
+
|
|
275
291
|
if not last_state:
|
|
276
292
|
err = f"[{task_name}] No result received from graph"
|
|
277
293
|
logger.warning(err)
|
|
@@ -302,12 +318,12 @@ class Agent:
|
|
|
302
318
|
self._finalize_tracing(task=task, context=context)
|
|
303
319
|
return output
|
|
304
320
|
|
|
305
|
-
def clean(self):
|
|
306
|
-
if not self._initialized:
|
|
321
|
+
def clean(self, force: bool = False):
|
|
322
|
+
if not self._initialized and not force:
|
|
307
323
|
return
|
|
308
324
|
screen_api_ok, hw_bridge_ok = stop_servers(
|
|
309
|
-
|
|
310
|
-
|
|
325
|
+
should_stop_screen_api=self._is_default_screen_api,
|
|
326
|
+
should_stop_hw_bridge=self._is_default_hw_bridge,
|
|
311
327
|
)
|
|
312
328
|
if not screen_api_ok:
|
|
313
329
|
logger.warning("Failed to stop Device Screen API.")
|
|
@@ -366,9 +382,9 @@ class Agent:
|
|
|
366
382
|
task_name: str,
|
|
367
383
|
ctx: MobileUseContext,
|
|
368
384
|
request: TaskRequest[TOutput],
|
|
369
|
-
output_config:
|
|
385
|
+
output_config: OutputConfig | None,
|
|
370
386
|
state: State,
|
|
371
|
-
) ->
|
|
387
|
+
) -> str | dict | TOutput | None:
|
|
372
388
|
if output_config and output_config.needs_structured_format():
|
|
373
389
|
logger.info(f"[{task_name}] Generating structured output...")
|
|
374
390
|
try:
|
|
@@ -402,10 +418,9 @@ class Agent:
|
|
|
402
418
|
focused_app_info=None,
|
|
403
419
|
device_date=None,
|
|
404
420
|
structured_decisions=None,
|
|
421
|
+
complete_subgoals_by_ids=[],
|
|
405
422
|
agents_thoughts=[],
|
|
406
423
|
remaining_steps=task.request.max_steps,
|
|
407
|
-
executor_retrigger=False,
|
|
408
|
-
executor_failed=False,
|
|
409
424
|
executor_messages=[],
|
|
410
425
|
cortex_last_thought=None,
|
|
411
426
|
)
|
|
@@ -2,19 +2,19 @@
|
|
|
2
2
|
Builder for AgentConfig objects using a fluent interface.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
from typing import Dict, Optional, List
|
|
6
5
|
import copy
|
|
7
6
|
|
|
7
|
+
from langchain_core.callbacks.base import Callbacks
|
|
8
|
+
|
|
8
9
|
from minitap.mobile_use.config import get_default_llm_config
|
|
10
|
+
from minitap.mobile_use.context import DevicePlatform
|
|
9
11
|
from minitap.mobile_use.sdk.constants import (
|
|
10
12
|
DEFAULT_HW_BRIDGE_BASE_URL,
|
|
11
13
|
DEFAULT_PROFILE_NAME,
|
|
12
14
|
DEFAULT_SCREEN_API_BASE_URL,
|
|
13
15
|
)
|
|
14
|
-
from minitap.mobile_use.sdk.types.agent import
|
|
15
|
-
from minitap.mobile_use.sdk.types.agent import AgentProfile
|
|
16
|
+
from minitap.mobile_use.sdk.types.agent import AgentConfig, AgentProfile, ApiBaseUrl, ServerConfig
|
|
16
17
|
from minitap.mobile_use.sdk.types.task import TaskRequestCommon
|
|
17
|
-
from minitap.mobile_use.context import DevicePlatform
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class AgentConfigBuilder:
|
|
@@ -38,12 +38,13 @@ class AgentConfigBuilder:
|
|
|
38
38
|
|
|
39
39
|
def __init__(self):
|
|
40
40
|
"""Initialize an empty AgentConfigBuilder."""
|
|
41
|
-
self._agent_profiles:
|
|
42
|
-
self._task_request_defaults:
|
|
43
|
-
self._default_profile:
|
|
44
|
-
self._device_id:
|
|
45
|
-
self._device_platform:
|
|
41
|
+
self._agent_profiles: dict[str, AgentProfile] = {}
|
|
42
|
+
self._task_request_defaults: TaskRequestCommon | None = None
|
|
43
|
+
self._default_profile: str | AgentProfile | None = None
|
|
44
|
+
self._device_id: str | None = None
|
|
45
|
+
self._device_platform: DevicePlatform | None = None
|
|
46
46
|
self._servers: ServerConfig = get_default_servers()
|
|
47
|
+
self._graph_config_callbacks: Callbacks = None
|
|
47
48
|
|
|
48
49
|
def add_profile(self, profile: AgentProfile) -> "AgentConfigBuilder":
|
|
49
50
|
"""
|
|
@@ -56,7 +57,7 @@ class AgentConfigBuilder:
|
|
|
56
57
|
profile.llm_config.validate_providers()
|
|
57
58
|
return self
|
|
58
59
|
|
|
59
|
-
def add_profiles(self, profiles:
|
|
60
|
+
def add_profiles(self, profiles: list[AgentProfile]) -> "AgentConfigBuilder":
|
|
60
61
|
"""
|
|
61
62
|
Add multiple agent profiles to the mobile-use agent.
|
|
62
63
|
|
|
@@ -128,7 +129,7 @@ class AgentConfigBuilder:
|
|
|
128
129
|
self._servers.screen_api_base_url = url
|
|
129
130
|
return self
|
|
130
131
|
|
|
131
|
-
def with_adb_server(self, host: str, port:
|
|
132
|
+
def with_adb_server(self, host: str, port: int | None = None) -> "AgentConfigBuilder":
|
|
132
133
|
"""
|
|
133
134
|
Set the ADB server host and port.
|
|
134
135
|
|
|
@@ -151,6 +152,16 @@ class AgentConfigBuilder:
|
|
|
151
152
|
self._servers = copy.deepcopy(servers)
|
|
152
153
|
return self
|
|
153
154
|
|
|
155
|
+
def with_graph_config_callbacks(self, callbacks: Callbacks) -> "AgentConfigBuilder":
|
|
156
|
+
"""
|
|
157
|
+
Set the graph config callbacks.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
callbacks: The graph config callbacks to use
|
|
161
|
+
"""
|
|
162
|
+
self._graph_config_callbacks = callbacks
|
|
163
|
+
return self
|
|
164
|
+
|
|
154
165
|
def build(self) -> AgentConfig:
|
|
155
166
|
"""
|
|
156
167
|
Build the mobile-use AgentConfig object.
|
|
@@ -197,6 +208,7 @@ class AgentConfigBuilder:
|
|
|
197
208
|
device_id=self._device_id,
|
|
198
209
|
device_platform=self._device_platform,
|
|
199
210
|
servers=self._servers,
|
|
211
|
+
graph_config_callbacks=self._graph_config_callbacks,
|
|
200
212
|
)
|
|
201
213
|
|
|
202
214
|
|
|
@@ -3,7 +3,7 @@ Builder for TaskRequest objects using a fluent interface.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import Self, TypeVar, cast
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel
|
|
9
9
|
|
|
@@ -12,7 +12,7 @@ from minitap.mobile_use.sdk.types.agent import AgentProfile
|
|
|
12
12
|
from minitap.mobile_use.sdk.types.task import TaskRequest, TaskRequestCommon
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
TIn = TypeVar("TIn", bound=
|
|
15
|
+
TIn = TypeVar("TIn", bound=BaseModel | None)
|
|
16
16
|
TOut = TypeVar("TOut", bound=BaseModel)
|
|
17
17
|
|
|
18
18
|
|
|
@@ -25,8 +25,8 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
25
25
|
self._max_steps = RECURSION_LIMIT
|
|
26
26
|
self._record_trace = False
|
|
27
27
|
self._trace_path = Path("mobile-use-traces")
|
|
28
|
-
self._llm_output_path:
|
|
29
|
-
self._thoughts_output_path:
|
|
28
|
+
self._llm_output_path: Path | None = None
|
|
29
|
+
self._thoughts_output_path: Path | None = None
|
|
30
30
|
|
|
31
31
|
def with_max_steps(self, max_steps: int) -> Self:
|
|
32
32
|
"""
|
|
@@ -38,7 +38,7 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
38
38
|
self._max_steps = max_steps
|
|
39
39
|
return self
|
|
40
40
|
|
|
41
|
-
def with_trace_recording(self, enabled: bool = True, path:
|
|
41
|
+
def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self:
|
|
42
42
|
"""
|
|
43
43
|
Configure trace recording for the task.
|
|
44
44
|
|
|
@@ -92,7 +92,7 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
92
92
|
)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
class TaskRequestBuilder
|
|
95
|
+
class TaskRequestBuilder[TIn](TaskRequestCommonBuilder):
|
|
96
96
|
"""
|
|
97
97
|
Builder class providing a fluent interface for creating TaskRequest objects.
|
|
98
98
|
|
|
@@ -114,10 +114,10 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
|
|
|
114
114
|
"""Initialize an empty TaskRequestBuilder."""
|
|
115
115
|
super().__init__()
|
|
116
116
|
self._goal = goal
|
|
117
|
-
self._profile:
|
|
118
|
-
self._name:
|
|
117
|
+
self._profile: str | AgentProfile | None = None
|
|
118
|
+
self._name: str | None = None
|
|
119
119
|
self._output_description = None
|
|
120
|
-
self._output_format:
|
|
120
|
+
self._output_format: type[TIn] | None = None
|
|
121
121
|
|
|
122
122
|
@classmethod
|
|
123
123
|
def from_common(cls, goal: str, common: TaskRequestCommon):
|
|
@@ -20,7 +20,6 @@ Run:
|
|
|
20
20
|
import asyncio
|
|
21
21
|
from datetime import datetime
|
|
22
22
|
from enum import Enum
|
|
23
|
-
from typing import List
|
|
24
23
|
|
|
25
24
|
from pydantic import BaseModel, Field
|
|
26
25
|
from minitap.mobile_use.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback
|
|
@@ -52,7 +51,7 @@ class NotificationSummary(BaseModel):
|
|
|
52
51
|
|
|
53
52
|
total_count: int = Field(..., description="Total number of notifications found")
|
|
54
53
|
high_priority_count: int = Field(0, description="Count of high priority notifications")
|
|
55
|
-
notifications:
|
|
54
|
+
notifications: list[Notification] = Field(
|
|
56
55
|
default_factory=list, description="List of individual notifications"
|
|
57
56
|
)
|
|
58
57
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Literal
|
|
2
2
|
from urllib.parse import urlparse
|
|
3
|
+
|
|
4
|
+
from langchain_core.callbacks.base import Callbacks
|
|
3
5
|
from pydantic import BaseModel
|
|
4
6
|
|
|
5
7
|
from minitap.mobile_use.context import DevicePlatform
|
|
@@ -13,7 +15,7 @@ class ApiBaseUrl(BaseModel):
|
|
|
13
15
|
|
|
14
16
|
scheme: Literal["http", "https"]
|
|
15
17
|
host: str
|
|
16
|
-
port:
|
|
18
|
+
port: int | None = None
|
|
17
19
|
|
|
18
20
|
def __eq__(self, other):
|
|
19
21
|
if not isinstance(other, ApiBaseUrl):
|
|
@@ -65,9 +67,12 @@ class AgentConfig(BaseModel):
|
|
|
65
67
|
servers: Custom server configurations.
|
|
66
68
|
"""
|
|
67
69
|
|
|
68
|
-
agent_profiles:
|
|
70
|
+
agent_profiles: dict[str, AgentProfile]
|
|
69
71
|
task_request_defaults: TaskRequestCommon
|
|
70
72
|
default_profile: AgentProfile
|
|
71
|
-
device_id:
|
|
72
|
-
device_platform:
|
|
73
|
+
device_id: str | None = None
|
|
74
|
+
device_platform: DevicePlatform | None = None
|
|
73
75
|
servers: ServerConfig
|
|
76
|
+
graph_config_callbacks: Callbacks = None
|
|
77
|
+
|
|
78
|
+
model_config = {"arbitrary_types_allowed": True}
|