minitap-mobile-use 2.0.1__py3-none-any.whl → 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of minitap-mobile-use might be problematic. Click here for more details.
- minitap/mobile_use/agents/cortex/cortex.md +7 -5
- minitap/mobile_use/agents/cortex/cortex.py +4 -1
- minitap/mobile_use/agents/cortex/types.py +1 -3
- minitap/mobile_use/agents/executor/executor.md +4 -5
- minitap/mobile_use/agents/executor/executor.py +3 -1
- minitap/mobile_use/agents/executor/tool_node.py +6 -6
- minitap/mobile_use/agents/outputter/outputter.py +1 -2
- minitap/mobile_use/agents/planner/planner.md +11 -2
- minitap/mobile_use/agents/planner/planner.py +7 -2
- minitap/mobile_use/agents/planner/types.py +3 -4
- minitap/mobile_use/agents/summarizer/summarizer.py +2 -1
- minitap/mobile_use/config.py +31 -16
- minitap/mobile_use/context.py +3 -4
- minitap/mobile_use/controllers/mobile_command_controller.py +36 -24
- minitap/mobile_use/controllers/platform_specific_commands_controller.py +3 -4
- minitap/mobile_use/graph/graph.py +1 -0
- minitap/mobile_use/graph/state.py +9 -9
- minitap/mobile_use/main.py +7 -8
- minitap/mobile_use/sdk/agent.py +25 -26
- minitap/mobile_use/sdk/builders/agent_config_builder.py +9 -10
- minitap/mobile_use/sdk/builders/task_request_builder.py +9 -9
- minitap/mobile_use/sdk/examples/smart_notification_assistant.py +1 -2
- minitap/mobile_use/sdk/types/agent.py +5 -5
- minitap/mobile_use/sdk/types/task.py +19 -18
- minitap/mobile_use/sdk/utils.py +4 -3
- minitap/mobile_use/servers/config.py +1 -2
- minitap/mobile_use/servers/device_hardware_bridge.py +3 -4
- minitap/mobile_use/servers/start_servers.py +4 -4
- minitap/mobile_use/servers/stop_servers.py +2 -3
- minitap/mobile_use/services/llm.py +24 -6
- minitap/mobile_use/tools/index.py +26 -14
- minitap/mobile_use/tools/mobile/back.py +1 -1
- minitap/mobile_use/tools/mobile/clear_text.py +277 -0
- minitap/mobile_use/tools/mobile/copy_text_from.py +1 -1
- minitap/mobile_use/tools/mobile/erase_one_char.py +56 -0
- minitap/mobile_use/tools/mobile/find_packages.py +1 -1
- minitap/mobile_use/tools/mobile/input_text.py +4 -80
- minitap/mobile_use/tools/mobile/launch_app.py +1 -1
- minitap/mobile_use/tools/mobile/long_press_on.py +2 -4
- minitap/mobile_use/tools/mobile/open_link.py +1 -1
- minitap/mobile_use/tools/mobile/paste_text.py +1 -1
- minitap/mobile_use/tools/mobile/press_key.py +1 -1
- minitap/mobile_use/tools/mobile/stop_app.py +2 -4
- minitap/mobile_use/tools/mobile/swipe.py +107 -9
- minitap/mobile_use/tools/mobile/take_screenshot.py +1 -1
- minitap/mobile_use/tools/mobile/tap.py +2 -4
- minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +2 -4
- minitap/mobile_use/tools/tool_wrapper.py +6 -1
- minitap/mobile_use/tools/utils.py +86 -0
- minitap/mobile_use/utils/cli_helpers.py +1 -2
- minitap/mobile_use/utils/cli_selection.py +5 -6
- minitap/mobile_use/utils/decorators.py +21 -20
- minitap/mobile_use/utils/logger.py +3 -4
- minitap/mobile_use/utils/media.py +1 -1
- minitap/mobile_use/utils/recorder.py +2 -9
- minitap/mobile_use/utils/ui_hierarchy.py +13 -5
- {minitap_mobile_use-2.0.1.dist-info → minitap_mobile_use-2.2.0.dist-info}/METADATA +35 -5
- minitap_mobile_use-2.2.0.dist-info/RECORD +96 -0
- minitap/mobile_use/tools/mobile/erase_text.py +0 -122
- minitap_mobile_use-2.0.1.dist-info/RECORD +0 -94
- {minitap_mobile_use-2.0.1.dist-info → minitap_mobile_use-2.2.0.dist-info}/WHEEL +0 -0
- {minitap_mobile_use-2.0.1.dist-info → minitap_mobile_use-2.2.0.dist-info}/entry_points.txt +0 -0
minitap/mobile_use/main.py
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import os
|
|
3
|
-
from typing import Optional
|
|
4
3
|
|
|
5
4
|
import typer
|
|
6
5
|
from adbutils import AdbClient
|
|
7
6
|
from langchain.callbacks.base import Callbacks
|
|
8
7
|
from rich.console import Console
|
|
9
|
-
from
|
|
8
|
+
from typing import Annotated
|
|
10
9
|
|
|
11
10
|
from minitap.mobile_use.config import (
|
|
12
11
|
initialize_llm_config,
|
|
@@ -24,9 +23,9 @@ logger = get_logger(__name__)
|
|
|
24
23
|
|
|
25
24
|
async def run_automation(
|
|
26
25
|
goal: str,
|
|
27
|
-
test_name:
|
|
26
|
+
test_name: str | None = None,
|
|
28
27
|
traces_output_path_str: str = "traces",
|
|
29
|
-
output_description:
|
|
28
|
+
output_description: str | None = None,
|
|
30
29
|
graph_config_callbacks: Callbacks = [],
|
|
31
30
|
):
|
|
32
31
|
llm_config = initialize_llm_config()
|
|
@@ -36,9 +35,9 @@ async def run_automation(
|
|
|
36
35
|
if settings.ADB_HOST:
|
|
37
36
|
config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT)
|
|
38
37
|
if settings.DEVICE_HARDWARE_BRIDGE_BASE_URL:
|
|
39
|
-
config.
|
|
38
|
+
config.with_hw_bridge(url=settings.DEVICE_HARDWARE_BRIDGE_BASE_URL)
|
|
40
39
|
if settings.DEVICE_SCREEN_API_BASE_URL:
|
|
41
|
-
config.
|
|
40
|
+
config.with_screen_api(url=settings.DEVICE_SCREEN_API_BASE_URL)
|
|
42
41
|
if graph_config_callbacks:
|
|
43
42
|
config.with_graph_config_callbacks(graph_config_callbacks)
|
|
44
43
|
|
|
@@ -70,7 +69,7 @@ async def run_automation(
|
|
|
70
69
|
def main(
|
|
71
70
|
goal: Annotated[str, typer.Argument(help="The main goal for the agent to achieve.")],
|
|
72
71
|
test_name: Annotated[
|
|
73
|
-
|
|
72
|
+
str | None,
|
|
74
73
|
typer.Option(
|
|
75
74
|
"--test-name",
|
|
76
75
|
"-n",
|
|
@@ -86,7 +85,7 @@ def main(
|
|
|
86
85
|
),
|
|
87
86
|
] = "traces",
|
|
88
87
|
output_description: Annotated[
|
|
89
|
-
|
|
88
|
+
str | None,
|
|
90
89
|
typer.Option(
|
|
91
90
|
"--output-description",
|
|
92
91
|
"-o",
|
minitap/mobile_use/sdk/agent.py
CHANGED
|
@@ -6,7 +6,7 @@ import uuid
|
|
|
6
6
|
from datetime import datetime
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from types import NoneType
|
|
9
|
-
from typing import
|
|
9
|
+
from typing import TypeVar, overload
|
|
10
10
|
|
|
11
11
|
from adbutils import AdbClient
|
|
12
12
|
from langchain_core.messages import AIMessage
|
|
@@ -61,7 +61,7 @@ from minitap.mobile_use.utils.recorder import log_agent_thought
|
|
|
61
61
|
|
|
62
62
|
logger = get_logger(__name__)
|
|
63
63
|
|
|
64
|
-
TOutput = TypeVar("TOutput", bound=
|
|
64
|
+
TOutput = TypeVar("TOutput", bound=BaseModel | None)
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class Agent:
|
|
@@ -74,9 +74,9 @@ class Agent:
|
|
|
74
74
|
_device_context: DeviceContext
|
|
75
75
|
_screen_api_client: ScreenApiClient
|
|
76
76
|
_hw_bridge_client: DeviceHardwareClient
|
|
77
|
-
_adb_client:
|
|
77
|
+
_adb_client: AdbClient | None
|
|
78
78
|
|
|
79
|
-
def __init__(self, config:
|
|
79
|
+
def __init__(self, config: AgentConfig | None = None):
|
|
80
80
|
self._config = config or get_default_agent_config()
|
|
81
81
|
self._tasks = []
|
|
82
82
|
self._tmp_traces_dir = Path(tempfile.gettempdir()) / "mobile-use-traces"
|
|
@@ -153,9 +153,9 @@ class Agent:
|
|
|
153
153
|
*,
|
|
154
154
|
goal: str,
|
|
155
155
|
output: type[TOutput],
|
|
156
|
-
profile:
|
|
157
|
-
name:
|
|
158
|
-
) ->
|
|
156
|
+
profile: str | AgentProfile | None = None,
|
|
157
|
+
name: str | None = None,
|
|
158
|
+
) -> TOutput | None: ...
|
|
159
159
|
|
|
160
160
|
@overload
|
|
161
161
|
async def run_task(
|
|
@@ -163,9 +163,9 @@ class Agent:
|
|
|
163
163
|
*,
|
|
164
164
|
goal: str,
|
|
165
165
|
output: str,
|
|
166
|
-
profile:
|
|
167
|
-
name:
|
|
168
|
-
) ->
|
|
166
|
+
profile: str | AgentProfile | None = None,
|
|
167
|
+
name: str | None = None,
|
|
168
|
+
) -> str | dict | None: ...
|
|
169
169
|
|
|
170
170
|
@overload
|
|
171
171
|
async def run_task(
|
|
@@ -173,25 +173,25 @@ class Agent:
|
|
|
173
173
|
*,
|
|
174
174
|
goal: str,
|
|
175
175
|
output=None,
|
|
176
|
-
profile:
|
|
177
|
-
name:
|
|
178
|
-
) ->
|
|
176
|
+
profile: str | AgentProfile | None = None,
|
|
177
|
+
name: str | None = None,
|
|
178
|
+
) -> str | None: ...
|
|
179
179
|
|
|
180
180
|
@overload
|
|
181
|
-
async def run_task(self, *, request: TaskRequest[None]) ->
|
|
181
|
+
async def run_task(self, *, request: TaskRequest[None]) -> str | dict | None: ...
|
|
182
182
|
|
|
183
183
|
@overload
|
|
184
|
-
async def run_task(self, *, request: TaskRequest[TOutput]) ->
|
|
184
|
+
async def run_task(self, *, request: TaskRequest[TOutput]) -> TOutput | None: ...
|
|
185
185
|
|
|
186
186
|
async def run_task(
|
|
187
187
|
self,
|
|
188
188
|
*,
|
|
189
|
-
goal:
|
|
190
|
-
output:
|
|
191
|
-
profile:
|
|
192
|
-
name:
|
|
193
|
-
request:
|
|
194
|
-
) ->
|
|
189
|
+
goal: str | None = None,
|
|
190
|
+
output: type[TOutput] | str | None = None,
|
|
191
|
+
profile: str | AgentProfile | None = None,
|
|
192
|
+
name: str | None = None,
|
|
193
|
+
request: TaskRequest[TOutput] | None = None,
|
|
194
|
+
) -> str | dict | TOutput | None:
|
|
195
195
|
if request is not None:
|
|
196
196
|
return await self._run_task(request)
|
|
197
197
|
if goal is None:
|
|
@@ -208,7 +208,7 @@ class Agent:
|
|
|
208
208
|
task_request.with_name(name=name)
|
|
209
209
|
return await self._run_task(task_request.build())
|
|
210
210
|
|
|
211
|
-
async def _run_task(self, request: TaskRequest[TOutput]) ->
|
|
211
|
+
async def _run_task(self, request: TaskRequest[TOutput]) -> str | dict | TOutput | None:
|
|
212
212
|
if not self._initialized:
|
|
213
213
|
raise AgentNotInitializedError()
|
|
214
214
|
|
|
@@ -278,13 +278,12 @@ class Agent:
|
|
|
278
278
|
)
|
|
279
279
|
|
|
280
280
|
if stream_mode == "updates":
|
|
281
|
-
for
|
|
281
|
+
for _, value in payload.items(): # type: ignore node name, node output
|
|
282
282
|
if value and "agents_thoughts" in value:
|
|
283
283
|
new_thoughts = value["agents_thoughts"]
|
|
284
284
|
last_item = new_thoughts[-1] if new_thoughts else None
|
|
285
285
|
if last_item:
|
|
286
286
|
log_agent_thought(
|
|
287
|
-
prefix=key,
|
|
288
287
|
agent_thought=last_item,
|
|
289
288
|
)
|
|
290
289
|
|
|
@@ -382,9 +381,9 @@ class Agent:
|
|
|
382
381
|
task_name: str,
|
|
383
382
|
ctx: MobileUseContext,
|
|
384
383
|
request: TaskRequest[TOutput],
|
|
385
|
-
output_config:
|
|
384
|
+
output_config: OutputConfig | None,
|
|
386
385
|
state: State,
|
|
387
|
-
) ->
|
|
386
|
+
) -> str | dict | TOutput | None:
|
|
388
387
|
if output_config and output_config.needs_structured_format():
|
|
389
388
|
logger.info(f"[{task_name}] Generating structured output...")
|
|
390
389
|
try:
|
|
@@ -3,7 +3,6 @@ Builder for AgentConfig objects using a fluent interface.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import copy
|
|
6
|
-
from typing import Dict, List, Optional
|
|
7
6
|
|
|
8
7
|
from langchain_core.callbacks.base import Callbacks
|
|
9
8
|
|
|
@@ -39,11 +38,11 @@ class AgentConfigBuilder:
|
|
|
39
38
|
|
|
40
39
|
def __init__(self):
|
|
41
40
|
"""Initialize an empty AgentConfigBuilder."""
|
|
42
|
-
self._agent_profiles:
|
|
43
|
-
self._task_request_defaults:
|
|
44
|
-
self._default_profile:
|
|
45
|
-
self._device_id:
|
|
46
|
-
self._device_platform:
|
|
41
|
+
self._agent_profiles: dict[str, AgentProfile] = {}
|
|
42
|
+
self._task_request_defaults: TaskRequestCommon | None = None
|
|
43
|
+
self._default_profile: str | AgentProfile | None = None
|
|
44
|
+
self._device_id: str | None = None
|
|
45
|
+
self._device_platform: DevicePlatform | None = None
|
|
47
46
|
self._servers: ServerConfig = get_default_servers()
|
|
48
47
|
self._graph_config_callbacks: Callbacks = None
|
|
49
48
|
|
|
@@ -58,7 +57,7 @@ class AgentConfigBuilder:
|
|
|
58
57
|
profile.llm_config.validate_providers()
|
|
59
58
|
return self
|
|
60
59
|
|
|
61
|
-
def add_profiles(self, profiles:
|
|
60
|
+
def add_profiles(self, profiles: list[AgentProfile]) -> "AgentConfigBuilder":
|
|
62
61
|
"""
|
|
63
62
|
Add multiple agent profiles to the mobile-use agent.
|
|
64
63
|
|
|
@@ -106,7 +105,7 @@ class AgentConfigBuilder:
|
|
|
106
105
|
self._task_request_defaults = copy.deepcopy(config)
|
|
107
106
|
return self
|
|
108
107
|
|
|
109
|
-
def
|
|
108
|
+
def with_hw_bridge(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
|
|
110
109
|
"""
|
|
111
110
|
Set the base URL for the device HW bridge API.
|
|
112
111
|
|
|
@@ -118,7 +117,7 @@ class AgentConfigBuilder:
|
|
|
118
117
|
self._servers.hw_bridge_base_url = url
|
|
119
118
|
return self
|
|
120
119
|
|
|
121
|
-
def
|
|
120
|
+
def with_screen_api(self, url: str | ApiBaseUrl) -> "AgentConfigBuilder":
|
|
122
121
|
"""
|
|
123
122
|
Set the base URL for the device screen API.
|
|
124
123
|
|
|
@@ -130,7 +129,7 @@ class AgentConfigBuilder:
|
|
|
130
129
|
self._servers.screen_api_base_url = url
|
|
131
130
|
return self
|
|
132
131
|
|
|
133
|
-
def with_adb_server(self, host: str, port:
|
|
132
|
+
def with_adb_server(self, host: str, port: int | None = None) -> "AgentConfigBuilder":
|
|
134
133
|
"""
|
|
135
134
|
Set the ADB server host and port.
|
|
136
135
|
|
|
@@ -3,7 +3,7 @@ Builder for TaskRequest objects using a fluent interface.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import Self, TypeVar, cast
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel
|
|
9
9
|
|
|
@@ -12,7 +12,7 @@ from minitap.mobile_use.sdk.types.agent import AgentProfile
|
|
|
12
12
|
from minitap.mobile_use.sdk.types.task import TaskRequest, TaskRequestCommon
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
TIn = TypeVar("TIn", bound=
|
|
15
|
+
TIn = TypeVar("TIn", bound=BaseModel | None)
|
|
16
16
|
TOut = TypeVar("TOut", bound=BaseModel)
|
|
17
17
|
|
|
18
18
|
|
|
@@ -25,8 +25,8 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
25
25
|
self._max_steps = RECURSION_LIMIT
|
|
26
26
|
self._record_trace = False
|
|
27
27
|
self._trace_path = Path("mobile-use-traces")
|
|
28
|
-
self._llm_output_path:
|
|
29
|
-
self._thoughts_output_path:
|
|
28
|
+
self._llm_output_path: Path | None = None
|
|
29
|
+
self._thoughts_output_path: Path | None = None
|
|
30
30
|
|
|
31
31
|
def with_max_steps(self, max_steps: int) -> Self:
|
|
32
32
|
"""
|
|
@@ -38,7 +38,7 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
38
38
|
self._max_steps = max_steps
|
|
39
39
|
return self
|
|
40
40
|
|
|
41
|
-
def with_trace_recording(self, enabled: bool = True, path:
|
|
41
|
+
def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self:
|
|
42
42
|
"""
|
|
43
43
|
Configure trace recording for the task.
|
|
44
44
|
|
|
@@ -92,7 +92,7 @@ class TaskRequestCommonBuilder(BaseModel):
|
|
|
92
92
|
)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
class TaskRequestBuilder
|
|
95
|
+
class TaskRequestBuilder[TIn](TaskRequestCommonBuilder):
|
|
96
96
|
"""
|
|
97
97
|
Builder class providing a fluent interface for creating TaskRequest objects.
|
|
98
98
|
|
|
@@ -114,10 +114,10 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
|
|
|
114
114
|
"""Initialize an empty TaskRequestBuilder."""
|
|
115
115
|
super().__init__()
|
|
116
116
|
self._goal = goal
|
|
117
|
-
self._profile:
|
|
118
|
-
self._name:
|
|
117
|
+
self._profile: str | AgentProfile | None = None
|
|
118
|
+
self._name: str | None = None
|
|
119
119
|
self._output_description = None
|
|
120
|
-
self._output_format:
|
|
120
|
+
self._output_format: type[TIn] | None = None
|
|
121
121
|
|
|
122
122
|
@classmethod
|
|
123
123
|
def from_common(cls, goal: str, common: TaskRequestCommon):
|
|
@@ -20,7 +20,6 @@ Run:
|
|
|
20
20
|
import asyncio
|
|
21
21
|
from datetime import datetime
|
|
22
22
|
from enum import Enum
|
|
23
|
-
from typing import List
|
|
24
23
|
|
|
25
24
|
from pydantic import BaseModel, Field
|
|
26
25
|
from minitap.mobile_use.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback
|
|
@@ -52,7 +51,7 @@ class NotificationSummary(BaseModel):
|
|
|
52
51
|
|
|
53
52
|
total_count: int = Field(..., description="Total number of notifications found")
|
|
54
53
|
high_priority_count: int = Field(0, description="Count of high priority notifications")
|
|
55
|
-
notifications:
|
|
54
|
+
notifications: list[Notification] = Field(
|
|
56
55
|
default_factory=list, description="List of individual notifications"
|
|
57
56
|
)
|
|
58
57
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Literal
|
|
2
2
|
from urllib.parse import urlparse
|
|
3
3
|
|
|
4
4
|
from langchain_core.callbacks.base import Callbacks
|
|
@@ -15,7 +15,7 @@ class ApiBaseUrl(BaseModel):
|
|
|
15
15
|
|
|
16
16
|
scheme: Literal["http", "https"]
|
|
17
17
|
host: str
|
|
18
|
-
port:
|
|
18
|
+
port: int | None = None
|
|
19
19
|
|
|
20
20
|
def __eq__(self, other):
|
|
21
21
|
if not isinstance(other, ApiBaseUrl):
|
|
@@ -67,11 +67,11 @@ class AgentConfig(BaseModel):
|
|
|
67
67
|
servers: Custom server configurations.
|
|
68
68
|
"""
|
|
69
69
|
|
|
70
|
-
agent_profiles:
|
|
70
|
+
agent_profiles: dict[str, AgentProfile]
|
|
71
71
|
task_request_defaults: TaskRequestCommon
|
|
72
72
|
default_profile: AgentProfile
|
|
73
|
-
device_id:
|
|
74
|
-
device_platform:
|
|
73
|
+
device_id: str | None = None
|
|
74
|
+
device_platform: DevicePlatform | None = None
|
|
75
75
|
servers: ServerConfig
|
|
76
76
|
graph_config_callbacks: Callbacks = None
|
|
77
77
|
|
|
@@ -5,7 +5,8 @@ Task-related type definitions for the Mobile-use SDK.
|
|
|
5
5
|
from datetime import datetime
|
|
6
6
|
from enum import Enum
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Any,
|
|
8
|
+
from typing import Any, TypeVar, overload
|
|
9
|
+
|
|
9
10
|
from pydantic import BaseModel, Field
|
|
10
11
|
|
|
11
12
|
from minitap.mobile_use.config import LLMConfig, get_default_llm_config
|
|
@@ -36,8 +37,8 @@ class AgentProfile(BaseModel):
|
|
|
36
37
|
self,
|
|
37
38
|
*,
|
|
38
39
|
name: str,
|
|
39
|
-
llm_config:
|
|
40
|
-
from_file:
|
|
40
|
+
llm_config: LLMConfig | None = None,
|
|
41
|
+
from_file: str | None = None,
|
|
41
42
|
**kwargs,
|
|
42
43
|
):
|
|
43
44
|
kwargs["name"] = name
|
|
@@ -64,7 +65,7 @@ class TaskStatus(str, Enum):
|
|
|
64
65
|
|
|
65
66
|
|
|
66
67
|
T = TypeVar("T", bound=BaseModel)
|
|
67
|
-
TOutput = TypeVar("TOutput", bound=
|
|
68
|
+
TOutput = TypeVar("TOutput", bound=BaseModel | None)
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
class TaskRequestCommon(BaseModel):
|
|
@@ -75,11 +76,11 @@ class TaskRequestCommon(BaseModel):
|
|
|
75
76
|
max_steps: int = RECURSION_LIMIT
|
|
76
77
|
record_trace: bool = False
|
|
77
78
|
trace_path: Path = Path("mobile-use-traces")
|
|
78
|
-
llm_output_path:
|
|
79
|
-
thoughts_output_path:
|
|
79
|
+
llm_output_path: Path | None = None
|
|
80
|
+
thoughts_output_path: Path | None = None
|
|
80
81
|
|
|
81
82
|
|
|
82
|
-
class TaskRequest
|
|
83
|
+
class TaskRequest[TOutput](TaskRequestCommon):
|
|
83
84
|
"""
|
|
84
85
|
Defines the format of a mobile automation task request.
|
|
85
86
|
|
|
@@ -98,10 +99,10 @@ class TaskRequest(TaskRequestCommon, Generic[TOutput]):
|
|
|
98
99
|
"""
|
|
99
100
|
|
|
100
101
|
goal: str
|
|
101
|
-
profile:
|
|
102
|
-
task_name:
|
|
103
|
-
output_description:
|
|
104
|
-
output_format:
|
|
102
|
+
profile: str | None = None
|
|
103
|
+
task_name: str | None = None
|
|
104
|
+
output_description: str | None = None
|
|
105
|
+
output_format: type[TOutput] | None = None
|
|
105
106
|
|
|
106
107
|
|
|
107
108
|
class TaskResult(BaseModel):
|
|
@@ -116,11 +117,11 @@ class TaskResult(BaseModel):
|
|
|
116
117
|
"""
|
|
117
118
|
|
|
118
119
|
content: Any = None
|
|
119
|
-
error:
|
|
120
|
+
error: str | None = None
|
|
120
121
|
execution_time_seconds: float
|
|
121
122
|
steps_taken: int
|
|
122
123
|
|
|
123
|
-
def get_as_model(self, model_class:
|
|
124
|
+
def get_as_model(self, model_class: type[T]) -> T:
|
|
124
125
|
"""
|
|
125
126
|
Parse the content into a Pydantic model instance.
|
|
126
127
|
|
|
@@ -158,14 +159,14 @@ class Task(BaseModel):
|
|
|
158
159
|
status: TaskStatus
|
|
159
160
|
request: TaskRequest
|
|
160
161
|
created_at: datetime
|
|
161
|
-
ended_at:
|
|
162
|
-
result:
|
|
162
|
+
ended_at: datetime | None = None
|
|
163
|
+
result: TaskResult | None = None
|
|
163
164
|
|
|
164
165
|
def finalize(
|
|
165
166
|
self,
|
|
166
|
-
content:
|
|
167
|
-
state:
|
|
168
|
-
error:
|
|
167
|
+
content: Any | None = None,
|
|
168
|
+
state: dict | None = None,
|
|
169
|
+
error: str | None = None,
|
|
169
170
|
cancelled: bool = False,
|
|
170
171
|
):
|
|
171
172
|
self.status = TaskStatus.COMPLETED if error is None else TaskStatus.FAILED
|
minitap/mobile_use/sdk/utils.py
CHANGED
|
@@ -2,11 +2,11 @@ import os
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
4
|
from pydantic import ValidationError
|
|
5
|
+
|
|
5
6
|
from minitap.mobile_use.config import LLMConfig, deep_merge_llm_config, get_default_llm_config
|
|
6
7
|
from minitap.mobile_use.utils.file import load_jsonc
|
|
7
8
|
from minitap.mobile_use.utils.logger import get_logger
|
|
8
9
|
|
|
9
|
-
|
|
10
10
|
logger = get_logger(__name__)
|
|
11
11
|
|
|
12
12
|
|
|
@@ -16,7 +16,7 @@ def load_llm_config_override(path: Path) -> LLMConfig:
|
|
|
16
16
|
override_config_dict = {}
|
|
17
17
|
if os.path.exists(path):
|
|
18
18
|
logger.info(f"Loading custom LLM config from {path.resolve()}...")
|
|
19
|
-
with open(path
|
|
19
|
+
with open(path) as f:
|
|
20
20
|
override_config_dict = load_jsonc(f)
|
|
21
21
|
else:
|
|
22
22
|
logger.warning("Custom LLM config not found - using the default config")
|
|
@@ -24,5 +24,6 @@ def load_llm_config_override(path: Path) -> LLMConfig:
|
|
|
24
24
|
try:
|
|
25
25
|
return deep_merge_llm_config(default_config, override_config_dict)
|
|
26
26
|
except ValidationError as e:
|
|
27
|
-
logger.error(f"Invalid LLM config: {e}
|
|
27
|
+
logger.error(f"Invalid LLM config: {e}")
|
|
28
|
+
logger.info("Falling back to default config")
|
|
28
29
|
return default_config
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
from typing import Optional
|
|
2
1
|
from dotenv import load_dotenv
|
|
3
2
|
from minitap.mobile_use.servers.device_hardware_bridge import DEVICE_HARDWARE_BRIDGE_PORT
|
|
4
3
|
from minitap.mobile_use.utils.logger import get_logger
|
|
@@ -11,7 +10,7 @@ logger = get_logger(__name__)
|
|
|
11
10
|
class ServerSettings(BaseSettings):
|
|
12
11
|
DEVICE_HARDWARE_BRIDGE_BASE_URL: str = f"http://localhost:{DEVICE_HARDWARE_BRIDGE_PORT}"
|
|
13
12
|
DEVICE_SCREEN_API_PORT: int = 9998
|
|
14
|
-
ADB_HOST:
|
|
13
|
+
ADB_HOST: str | None = None
|
|
15
14
|
|
|
16
15
|
model_config = {"env_file": ".env", "extra": "ignore"}
|
|
17
16
|
|
|
@@ -4,7 +4,6 @@ import subprocess
|
|
|
4
4
|
import threading
|
|
5
5
|
import time
|
|
6
6
|
from enum import Enum
|
|
7
|
-
from typing import Optional
|
|
8
7
|
|
|
9
8
|
import requests
|
|
10
9
|
from minitap.mobile_use.context import DevicePlatform
|
|
@@ -24,7 +23,7 @@ class BridgeStatus(Enum):
|
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
class DeviceHardwareBridge:
|
|
27
|
-
def __init__(self, device_id: str, platform: DevicePlatform, adb_host:
|
|
26
|
+
def __init__(self, device_id: str, platform: DevicePlatform, adb_host: str | None = None):
|
|
28
27
|
self.process = None
|
|
29
28
|
self.status = BridgeStatus.STOPPED
|
|
30
29
|
self.thread = None
|
|
@@ -32,7 +31,7 @@ class DeviceHardwareBridge:
|
|
|
32
31
|
self.lock = threading.Lock()
|
|
33
32
|
self.device_id: str = device_id
|
|
34
33
|
self.platform: DevicePlatform = platform
|
|
35
|
-
self.adb_host:
|
|
34
|
+
self.adb_host: str | None = adb_host
|
|
36
35
|
|
|
37
36
|
def _run_maestro_studio(self):
|
|
38
37
|
try:
|
|
@@ -207,6 +206,6 @@ class DeviceHardwareBridge:
|
|
|
207
206
|
with self.lock:
|
|
208
207
|
return {"status": self.status.value, "output": self.output[-10:]}
|
|
209
208
|
|
|
210
|
-
def get_device_id(self) ->
|
|
209
|
+
def get_device_id(self) -> str | None:
|
|
211
210
|
with self.lock:
|
|
212
211
|
return self.device_id
|
|
@@ -4,7 +4,7 @@ import signal
|
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
6
|
from enum import Enum
|
|
7
|
-
from typing import Annotated
|
|
7
|
+
from typing import Annotated
|
|
8
8
|
|
|
9
9
|
import requests
|
|
10
10
|
import typer
|
|
@@ -22,7 +22,7 @@ bridge_instance = None
|
|
|
22
22
|
shutdown_requested = False
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
def check_device_screen_api_health(base_url:
|
|
25
|
+
def check_device_screen_api_health(base_url: str | None = None, max_retries=30, delay=1):
|
|
26
26
|
base_url = base_url or f"http://localhost:{server_settings.DEVICE_SCREEN_API_PORT}"
|
|
27
27
|
health_url = f"{base_url}/health"
|
|
28
28
|
|
|
@@ -49,7 +49,7 @@ def check_device_screen_api_health(base_url: Optional[str] = None, max_retries=3
|
|
|
49
49
|
return False
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
def _start_device_screen_api_process() ->
|
|
52
|
+
def _start_device_screen_api_process() -> multiprocessing.Process | None:
|
|
53
53
|
try:
|
|
54
54
|
process = multiprocessing.Process(target=start_device_screen_api, daemon=True)
|
|
55
55
|
process.start()
|
|
@@ -61,7 +61,7 @@ def _start_device_screen_api_process() -> Optional[multiprocessing.Process]:
|
|
|
61
61
|
|
|
62
62
|
def start_device_hardware_bridge(
|
|
63
63
|
device_id: str, platform: DevicePlatform
|
|
64
|
-
) ->
|
|
64
|
+
) -> DeviceHardwareBridge | None:
|
|
65
65
|
logger.info("Starting Device Hardware Bridge...")
|
|
66
66
|
|
|
67
67
|
try:
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
import time
|
|
3
|
-
from typing import List
|
|
4
3
|
|
|
5
4
|
import psutil
|
|
6
5
|
import requests
|
|
@@ -12,7 +11,7 @@ from minitap.mobile_use.utils.logger import get_server_logger
|
|
|
12
11
|
logger = get_server_logger()
|
|
13
12
|
|
|
14
13
|
|
|
15
|
-
def find_processes_by_name(name: str) ->
|
|
14
|
+
def find_processes_by_name(name: str) -> list[psutil.Process]:
|
|
16
15
|
"""Find all processes with the given name."""
|
|
17
16
|
processes = []
|
|
18
17
|
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
|
|
@@ -26,7 +25,7 @@ def find_processes_by_name(name: str) -> List[psutil.Process]:
|
|
|
26
25
|
return processes
|
|
27
26
|
|
|
28
27
|
|
|
29
|
-
def find_processes_by_port(port: int) ->
|
|
28
|
+
def find_processes_by_port(port: int) -> list[psutil.Process]:
|
|
30
29
|
processes = []
|
|
31
30
|
for proc in psutil.process_iter(["pid", "name"]):
|
|
32
31
|
try:
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from
|
|
3
|
-
from
|
|
2
|
+
from collections.abc import Awaitable, Callable
|
|
3
|
+
from typing import Literal, TypeVar, overload
|
|
4
4
|
|
|
5
|
+
from langchain_core.language_models.chat_models import BaseChatModel
|
|
5
6
|
from langchain_google_genai import ChatGoogleGenerativeAI
|
|
7
|
+
from langchain_google_vertexai import ChatVertexAI
|
|
6
8
|
from langchain_openai import ChatOpenAI
|
|
9
|
+
|
|
7
10
|
from minitap.mobile_use.config import (
|
|
8
11
|
AgentNode,
|
|
9
12
|
AgentNodeWithFallback,
|
|
@@ -31,6 +34,19 @@ def get_google_llm(
|
|
|
31
34
|
return client
|
|
32
35
|
|
|
33
36
|
|
|
37
|
+
def get_vertex_llm(
|
|
38
|
+
model_name: str = "gemini-2.5-pro",
|
|
39
|
+
temperature: float = 0.7,
|
|
40
|
+
) -> ChatVertexAI:
|
|
41
|
+
client = ChatVertexAI(
|
|
42
|
+
model_name=model_name,
|
|
43
|
+
max_tokens=None,
|
|
44
|
+
temperature=temperature,
|
|
45
|
+
max_retries=2,
|
|
46
|
+
)
|
|
47
|
+
return client
|
|
48
|
+
|
|
49
|
+
|
|
34
50
|
def get_openai_llm(
|
|
35
51
|
model_name: str = "o3",
|
|
36
52
|
temperature: float = 1,
|
|
@@ -74,7 +90,7 @@ def get_llm(
|
|
|
74
90
|
*,
|
|
75
91
|
use_fallback: bool = False,
|
|
76
92
|
temperature: float = 1,
|
|
77
|
-
): ...
|
|
93
|
+
) -> BaseChatModel: ...
|
|
78
94
|
|
|
79
95
|
|
|
80
96
|
@overload
|
|
@@ -83,7 +99,7 @@ def get_llm(
|
|
|
83
99
|
name: AgentNode,
|
|
84
100
|
*,
|
|
85
101
|
temperature: float = 1,
|
|
86
|
-
): ...
|
|
102
|
+
) -> BaseChatModel: ...
|
|
87
103
|
|
|
88
104
|
|
|
89
105
|
@overload
|
|
@@ -93,7 +109,7 @@ def get_llm(
|
|
|
93
109
|
*,
|
|
94
110
|
is_utils: Literal[True],
|
|
95
111
|
temperature: float = 1,
|
|
96
|
-
): ...
|
|
112
|
+
) -> BaseChatModel: ...
|
|
97
113
|
|
|
98
114
|
|
|
99
115
|
def get_llm(
|
|
@@ -102,7 +118,7 @@ def get_llm(
|
|
|
102
118
|
is_utils: bool = False,
|
|
103
119
|
use_fallback: bool = False,
|
|
104
120
|
temperature: float = 1,
|
|
105
|
-
):
|
|
121
|
+
) -> BaseChatModel:
|
|
106
122
|
llm = (
|
|
107
123
|
ctx.llm_config.get_utils(name) # type: ignore
|
|
108
124
|
if is_utils
|
|
@@ -117,6 +133,8 @@ def get_llm(
|
|
|
117
133
|
return get_openai_llm(llm.model, temperature)
|
|
118
134
|
elif llm.provider == "google":
|
|
119
135
|
return get_google_llm(llm.model, temperature)
|
|
136
|
+
elif llm.provider == "vertexai":
|
|
137
|
+
return get_vertex_llm(llm.model, temperature)
|
|
120
138
|
elif llm.provider == "openrouter":
|
|
121
139
|
return get_openrouter_llm(llm.model, temperature)
|
|
122
140
|
elif llm.provider == "xai":
|