minitap-mobile-use 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of minitap-mobile-use might be problematic. Click here for more details.

Files changed (55) hide show
  1. minitap/mobile_use/agents/cortex/cortex.md +17 -10
  2. minitap/mobile_use/agents/cortex/cortex.py +12 -2
  3. minitap/mobile_use/agents/cortex/types.py +2 -2
  4. minitap/mobile_use/agents/executor/executor.md +16 -10
  5. minitap/mobile_use/agents/executor/executor.py +6 -18
  6. minitap/mobile_use/agents/executor/tool_node.py +105 -0
  7. minitap/mobile_use/agents/hopper/hopper.md +2 -10
  8. minitap/mobile_use/agents/hopper/hopper.py +4 -9
  9. minitap/mobile_use/agents/orchestrator/human.md +3 -4
  10. minitap/mobile_use/agents/orchestrator/orchestrator.md +25 -7
  11. minitap/mobile_use/agents/orchestrator/orchestrator.py +56 -56
  12. minitap/mobile_use/agents/orchestrator/types.py +5 -8
  13. minitap/mobile_use/agents/planner/planner.md +14 -13
  14. minitap/mobile_use/agents/planner/planner.py +4 -1
  15. minitap/mobile_use/agents/planner/types.py +8 -2
  16. minitap/mobile_use/agents/planner/utils.py +11 -0
  17. minitap/mobile_use/clients/device_hardware_client.py +3 -0
  18. minitap/mobile_use/config.py +2 -0
  19. minitap/mobile_use/constants.py +1 -0
  20. minitap/mobile_use/controllers/mobile_command_controller.py +10 -11
  21. minitap/mobile_use/graph/graph.py +9 -31
  22. minitap/mobile_use/graph/state.py +26 -6
  23. minitap/mobile_use/main.py +6 -2
  24. minitap/mobile_use/sdk/agent.py +54 -39
  25. minitap/mobile_use/sdk/builders/agent_config_builder.py +17 -4
  26. minitap/mobile_use/sdk/types/agent.py +5 -0
  27. minitap/mobile_use/servers/stop_servers.py +10 -15
  28. minitap/mobile_use/services/llm.py +1 -0
  29. minitap/mobile_use/tools/index.py +2 -4
  30. minitap/mobile_use/tools/mobile/back.py +7 -11
  31. minitap/mobile_use/tools/mobile/copy_text_from.py +7 -11
  32. minitap/mobile_use/tools/mobile/erase_text.py +7 -9
  33. minitap/mobile_use/tools/mobile/find_packages.py +69 -0
  34. minitap/mobile_use/tools/mobile/input_text.py +131 -32
  35. minitap/mobile_use/tools/mobile/launch_app.py +7 -11
  36. minitap/mobile_use/tools/mobile/long_press_on.py +7 -9
  37. minitap/mobile_use/tools/mobile/open_link.py +7 -11
  38. minitap/mobile_use/tools/mobile/paste_text.py +7 -11
  39. minitap/mobile_use/tools/mobile/press_key.py +7 -11
  40. minitap/mobile_use/tools/mobile/stop_app.py +7 -9
  41. minitap/mobile_use/tools/mobile/swipe.py +7 -11
  42. minitap/mobile_use/tools/mobile/take_screenshot.py +7 -11
  43. minitap/mobile_use/tools/mobile/tap.py +7 -9
  44. minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +7 -9
  45. minitap/mobile_use/tools/tool_wrapper.py +1 -23
  46. minitap/mobile_use/utils/recorder.py +11 -10
  47. minitap/mobile_use/utils/ui_hierarchy.py +88 -1
  48. {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.0.1.dist-info}/METADATA +2 -2
  49. minitap_mobile_use-2.0.1.dist-info/RECORD +94 -0
  50. minitap/mobile_use/agents/executor/executor_context_cleaner.py +0 -27
  51. minitap/mobile_use/tools/mobile/list_packages.py +0 -78
  52. minitap/mobile_use/tools/mobile/run_flow.py +0 -57
  53. minitap_mobile_use-2.0.0.dist-info/RECORD +0 -95
  54. {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.0.1.dist-info}/WHEEL +0 -0
  55. {minitap_mobile_use-2.0.0.dist-info → minitap_mobile_use-2.0.1.dist-info}/entry_points.txt +0 -0
@@ -1,16 +1,15 @@
1
- from typing import Optional
2
-
3
1
  from langchain_core.messages import ToolMessage
4
2
  from langchain_core.tools import tool
5
3
  from langchain_core.tools.base import InjectedToolCallId
6
4
  from langgraph.types import Command
5
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
7
6
  from minitap.mobile_use.context import MobileUseContext
8
7
  from minitap.mobile_use.controllers.mobile_command_controller import (
9
8
  paste_text as paste_text_controller,
10
9
  )
11
10
  from minitap.mobile_use.graph.state import State
12
11
  from langgraph.prebuilt import InjectedState
13
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
12
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
14
13
  from typing_extensions import Annotated
15
14
 
16
15
 
@@ -20,7 +19,6 @@ def get_paste_text_tool(ctx: MobileUseContext):
20
19
  tool_call_id: Annotated[str, InjectedToolCallId],
21
20
  state: Annotated[State, InjectedState],
22
21
  agent_thought: str,
23
- executor_metadata: Optional[ExecutorMetadata],
24
22
  ):
25
23
  """
26
24
  Pastes text previously copied via `copyTextFrom` into the currently focused field.
@@ -41,18 +39,16 @@ def get_paste_text_tool(ctx: MobileUseContext):
41
39
  if has_failed
42
40
  else paste_text_wrapper.on_success_fn(),
43
41
  additional_kwargs={"error": output} if has_failed else {},
42
+ status="error" if has_failed else "success",
44
43
  )
45
44
  return Command(
46
- update=paste_text_wrapper.handle_executor_state_fields(
45
+ update=state.sanitize_update(
47
46
  ctx=ctx,
48
- state=state,
49
- executor_metadata=executor_metadata,
50
- tool_message=tool_message,
51
- is_failure=has_failed,
52
- updates={
47
+ update={
53
48
  "agents_thoughts": [agent_thought],
54
- "messages": [tool_message],
49
+ EXECUTOR_MESSAGES_KEY: [tool_message],
55
50
  },
51
+ agent="executor",
56
52
  ),
57
53
  )
58
54
 
@@ -1,17 +1,16 @@
1
- from typing import Optional
2
-
3
1
  from langchain_core.messages import ToolMessage
4
2
  from langchain_core.tools import tool
5
3
  from langchain_core.tools.base import InjectedToolCallId
6
4
  from langgraph.prebuilt import InjectedState
7
5
  from langgraph.types import Command
6
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
7
  from minitap.mobile_use.context import MobileUseContext
9
8
  from minitap.mobile_use.controllers.mobile_command_controller import Key
10
9
  from minitap.mobile_use.controllers.mobile_command_controller import (
11
10
  press_key as press_key_controller,
12
11
  )
13
12
  from minitap.mobile_use.graph.state import State
14
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
13
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
15
14
  from typing_extensions import Annotated
16
15
 
17
16
 
@@ -21,7 +20,6 @@ def get_press_key_tool(ctx: MobileUseContext):
21
20
  tool_call_id: Annotated[str, InjectedToolCallId],
22
21
  state: Annotated[State, InjectedState],
23
22
  agent_thought: str,
24
- executor_metadata: Optional[ExecutorMetadata],
25
23
  key: Key,
26
24
  ):
27
25
  """Press a key on the device."""
@@ -33,18 +31,16 @@ def get_press_key_tool(ctx: MobileUseContext):
33
31
  if has_failed
34
32
  else press_key_wrapper.on_success_fn(key),
35
33
  additional_kwargs={"error": output} if has_failed else {},
34
+ status="error" if has_failed else "success",
36
35
  )
37
36
  return Command(
38
- update=press_key_wrapper.handle_executor_state_fields(
37
+ update=state.sanitize_update(
39
38
  ctx=ctx,
40
- state=state,
41
- executor_metadata=executor_metadata,
42
- tool_message=tool_message,
43
- is_failure=has_failed,
44
- updates={
39
+ update={
45
40
  "agents_thoughts": [agent_thought],
46
- "messages": [tool_message],
41
+ EXECUTOR_MESSAGES_KEY: [tool_message],
47
42
  },
43
+ agent="executor",
48
44
  ),
49
45
  )
50
46
 
@@ -5,10 +5,11 @@ from langchain_core.tools import tool
5
5
  from langchain_core.tools.base import InjectedToolCallId
6
6
  from langgraph.prebuilt import InjectedState
7
7
  from langgraph.types import Command
8
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
9
  from minitap.mobile_use.context import MobileUseContext
9
10
  from minitap.mobile_use.controllers.mobile_command_controller import stop_app as stop_app_controller
10
11
  from minitap.mobile_use.graph.state import State
11
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
12
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
12
13
  from typing_extensions import Annotated
13
14
 
14
15
 
@@ -18,7 +19,6 @@ def get_stop_app_tool(ctx: MobileUseContext):
18
19
  tool_call_id: Annotated[str, InjectedToolCallId],
19
20
  state: Annotated[State, InjectedState],
20
21
  agent_thought: str,
21
- executor_metadata: Optional[ExecutorMetadata],
22
22
  package_name: Optional[str] = None,
23
23
  ):
24
24
  """
@@ -33,18 +33,16 @@ def get_stop_app_tool(ctx: MobileUseContext):
33
33
  if has_failed
34
34
  else stop_app_wrapper.on_success_fn(package_name),
35
35
  additional_kwargs={"error": output} if has_failed else {},
36
+ status="error" if has_failed else "success",
36
37
  )
37
38
  return Command(
38
- update=stop_app_wrapper.handle_executor_state_fields(
39
+ update=state.sanitize_update(
39
40
  ctx=ctx,
40
- state=state,
41
- executor_metadata=executor_metadata,
42
- tool_message=tool_message,
43
- is_failure=has_failed,
44
- updates={
41
+ update={
45
42
  "agents_thoughts": [agent_thought],
46
- "messages": [tool_message],
43
+ EXECUTOR_MESSAGES_KEY: [tool_message],
47
44
  },
45
+ agent="executor",
48
46
  ),
49
47
  )
50
48
 
@@ -1,15 +1,14 @@
1
- from typing import Optional
2
-
3
1
  from langchain_core.messages import ToolMessage
4
2
  from langchain_core.tools import tool
5
3
  from langchain_core.tools.base import InjectedToolCallId
6
4
  from langgraph.prebuilt import InjectedState
7
5
  from langgraph.types import Command
6
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
7
  from minitap.mobile_use.context import MobileUseContext
9
8
  from minitap.mobile_use.controllers.mobile_command_controller import SwipeRequest
10
9
  from minitap.mobile_use.controllers.mobile_command_controller import swipe as swipe_controller
11
10
  from minitap.mobile_use.graph.state import State
12
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
11
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
13
12
  from typing_extensions import Annotated
14
13
 
15
14
 
@@ -19,7 +18,6 @@ def get_swipe_tool(ctx: MobileUseContext):
19
18
  tool_call_id: Annotated[str, InjectedToolCallId],
20
19
  state: Annotated[State, InjectedState],
21
20
  agent_thought: str,
22
- executor_metadata: Optional[ExecutorMetadata],
23
21
  swipe_request: SwipeRequest,
24
22
  ):
25
23
  """
@@ -31,18 +29,16 @@ def get_swipe_tool(ctx: MobileUseContext):
31
29
  tool_call_id=tool_call_id,
32
30
  content=swipe_wrapper.on_failure_fn() if has_failed else swipe_wrapper.on_success_fn(),
33
31
  additional_kwargs={"error": output} if has_failed else {},
32
+ status="error" if has_failed else "success",
34
33
  )
35
34
  return Command(
36
- update=swipe_wrapper.handle_executor_state_fields(
35
+ update=state.sanitize_update(
37
36
  ctx=ctx,
38
- state=state,
39
- executor_metadata=executor_metadata,
40
- tool_message=tool_message,
41
- is_failure=has_failed,
42
- updates={
37
+ update={
43
38
  "agents_thoughts": [agent_thought],
44
- "messages": [tool_message],
39
+ EXECUTOR_MESSAGES_KEY: [tool_message],
45
40
  },
41
+ agent="executor",
46
42
  ),
47
43
  )
48
44
 
@@ -1,16 +1,15 @@
1
- from typing import Optional
2
-
3
1
  from langchain_core.messages import ToolMessage
4
2
  from langchain_core.tools import tool
5
3
  from langchain_core.tools.base import InjectedToolCallId
6
4
  from langgraph.prebuilt import InjectedState
7
5
  from langgraph.types import Command
6
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
7
  from minitap.mobile_use.context import MobileUseContext
9
8
  from minitap.mobile_use.controllers.mobile_command_controller import (
10
9
  take_screenshot as take_screenshot_controller,
11
10
  )
12
11
  from minitap.mobile_use.graph.state import State
13
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
12
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
14
13
  from minitap.mobile_use.utils.media import compress_base64_jpeg
15
14
  from typing_extensions import Annotated
16
15
 
@@ -21,7 +20,6 @@ def get_take_screenshot_tool(ctx: MobileUseContext):
21
20
  tool_call_id: Annotated[str, InjectedToolCallId],
22
21
  state: Annotated[State, InjectedState],
23
22
  agent_thought: str,
24
- executor_metadata: Optional[ExecutorMetadata],
25
23
  ):
26
24
  """
27
25
  Take a screenshot of the device.
@@ -42,21 +40,19 @@ def get_take_screenshot_tool(ctx: MobileUseContext):
42
40
  if has_failed
43
41
  else take_screenshot_wrapper.on_success_fn(),
44
42
  additional_kwargs={"error": output} if has_failed else {},
43
+ status="error" if has_failed else "success",
45
44
  )
46
45
  updates = {
47
46
  "agents_thoughts": [agent_thought],
48
- "messages": [tool_message],
47
+ EXECUTOR_MESSAGES_KEY: [tool_message],
49
48
  }
50
49
  if compressed_image_base64:
51
50
  updates["latest_screenshot_base64"] = compressed_image_base64
52
51
  return Command(
53
- update=take_screenshot_wrapper.handle_executor_state_fields(
52
+ update=state.sanitize_update(
54
53
  ctx=ctx,
55
- state=state,
56
- executor_metadata=executor_metadata,
57
- tool_message=tool_message,
58
- is_failure=has_failed,
59
- updates=updates,
54
+ update=updates,
55
+ agent="executor",
60
56
  ),
61
57
  )
62
58
 
@@ -5,11 +5,12 @@ from langchain_core.tools import tool
5
5
  from langchain_core.tools.base import InjectedToolCallId
6
6
  from langgraph.prebuilt import InjectedState
7
7
  from langgraph.types import Command
8
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
9
  from minitap.mobile_use.context import MobileUseContext
9
10
  from minitap.mobile_use.controllers.mobile_command_controller import SelectorRequest
10
11
  from minitap.mobile_use.controllers.mobile_command_controller import tap as tap_controller
11
12
  from minitap.mobile_use.graph.state import State
12
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
13
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
13
14
  from typing_extensions import Annotated
14
15
 
15
16
 
@@ -19,7 +20,6 @@ def get_tap_tool(ctx: MobileUseContext):
19
20
  tool_call_id: Annotated[str, InjectedToolCallId],
20
21
  state: Annotated[State, InjectedState],
21
22
  agent_thought: str,
22
- executor_metadata: Optional[ExecutorMetadata],
23
23
  selector_request: SelectorRequest,
24
24
  index: Optional[int] = None,
25
25
  ):
@@ -35,18 +35,16 @@ def get_tap_tool(ctx: MobileUseContext):
35
35
  if has_failed
36
36
  else tap_wrapper.on_success_fn(selector_request, index),
37
37
  additional_kwargs={"error": output} if has_failed else {},
38
+ status="error" if has_failed else "success",
38
39
  )
39
40
  return Command(
40
- update=tap_wrapper.handle_executor_state_fields(
41
+ update=state.sanitize_update(
41
42
  ctx=ctx,
42
- state=state,
43
- executor_metadata=executor_metadata,
44
- tool_message=tool_message,
45
- is_failure=has_failed,
46
- updates={
43
+ update={
47
44
  "agents_thoughts": [agent_thought],
48
- "messages": [tool_message],
45
+ EXECUTOR_MESSAGES_KEY: [tool_message],
49
46
  },
47
+ agent="executor",
50
48
  ),
51
49
  )
52
50
 
@@ -5,13 +5,14 @@ from langchain_core.tools import tool
5
5
  from langchain_core.tools.base import InjectedToolCallId
6
6
  from langgraph.prebuilt import InjectedState
7
7
  from langgraph.types import Command
8
+ from minitap.mobile_use.constants import EXECUTOR_MESSAGES_KEY
8
9
  from minitap.mobile_use.context import MobileUseContext
9
10
  from minitap.mobile_use.controllers.mobile_command_controller import WaitTimeout
10
11
  from minitap.mobile_use.controllers.mobile_command_controller import (
11
12
  wait_for_animation_to_end as wait_for_animation_to_end_controller,
12
13
  )
13
14
  from minitap.mobile_use.graph.state import State
14
- from minitap.mobile_use.tools.tool_wrapper import ExecutorMetadata, ToolWrapper
15
+ from minitap.mobile_use.tools.tool_wrapper import ToolWrapper
15
16
  from typing_extensions import Annotated
16
17
 
17
18
 
@@ -21,7 +22,6 @@ def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
21
22
  tool_call_id: Annotated[str, InjectedToolCallId],
22
23
  state: Annotated[State, InjectedState],
23
24
  agent_thought: str,
24
- executor_metadata: Optional[ExecutorMetadata],
25
25
  timeout: Optional[WaitTimeout],
26
26
  ):
27
27
  """
@@ -43,18 +43,16 @@ def get_wait_for_animation_to_end_tool(ctx: MobileUseContext):
43
43
  if has_failed
44
44
  else wait_for_animation_to_end_wrapper.on_success_fn(timeout),
45
45
  additional_kwargs={"error": output} if has_failed else {},
46
+ status="error" if has_failed else "success",
46
47
  )
47
48
  return Command(
48
- update=wait_for_animation_to_end_wrapper.handle_executor_state_fields(
49
+ update=state.sanitize_update(
49
50
  ctx=ctx,
50
- state=state,
51
- executor_metadata=executor_metadata,
52
- tool_message=tool_message,
53
- is_failure=has_failed,
54
- updates={
51
+ update={
55
52
  "agents_thoughts": [agent_thought],
56
- "messages": [tool_message],
53
+ EXECUTOR_MESSAGES_KEY: [tool_message],
57
54
  },
55
+ agent="executor",
58
56
  ),
59
57
  )
60
58
 
@@ -1,33 +1,11 @@
1
- from typing import Callable, Optional
1
+ from typing import Callable
2
2
 
3
- from langchain_core.messages import ToolMessage
4
3
  from langchain_core.tools import BaseTool
5
4
  from pydantic import BaseModel
6
5
  from minitap.mobile_use.context import MobileUseContext
7
- from minitap.mobile_use.graph.state import State
8
-
9
-
10
- class ExecutorMetadata(BaseModel):
11
- retrigger: bool
12
6
 
13
7
 
14
8
  class ToolWrapper(BaseModel):
15
9
  tool_fn_getter: Callable[[MobileUseContext], BaseTool]
16
10
  on_success_fn: Callable[..., str]
17
11
  on_failure_fn: Callable[..., str]
18
-
19
- def handle_executor_state_fields(
20
- self,
21
- ctx: MobileUseContext,
22
- state: State,
23
- executor_metadata: Optional[ExecutorMetadata],
24
- is_failure: bool,
25
- tool_message: ToolMessage,
26
- updates: dict,
27
- ):
28
- if executor_metadata is None:
29
- return state.sanitize_update(ctx=ctx, update=updates)
30
- updates["executor_retrigger"] = executor_metadata.retrigger
31
- updates["executor_messages"] = [tool_message]
32
- updates["executor_failed"] = is_failure
33
- return state.sanitize_update(ctx=ctx, update=updates)
@@ -1,9 +1,9 @@
1
1
  import base64
2
2
  import time
3
- from pathlib import Path
4
3
 
4
+ from colorama import Fore, Style
5
5
  from langchain_core.messages import BaseMessage
6
- from minitap.mobile_use.config import record_events
6
+
7
7
  from minitap.mobile_use.context import MobileUseContext
8
8
  from minitap.mobile_use.controllers.mobile_command_controller import take_screenshot
9
9
  from minitap.mobile_use.utils.logger import get_logger
@@ -45,11 +45,12 @@ def record_interaction(ctx: MobileUseContext, response: BaseMessage):
45
45
  return "Screenshot recorded successfully"
46
46
 
47
47
 
48
- def log_agent_thoughts(agents_thoughts: list[str], output_path: Path | None):
49
- if len(agents_thoughts) > 0:
50
- last_agents_thoughts = agents_thoughts[-1]
51
- previous_last_agents_thoughts = agents_thoughts[-2] if len(agents_thoughts) > 1 else None
52
- if previous_last_agents_thoughts != last_agents_thoughts:
53
- logger.info(f"💭 {last_agents_thoughts}")
54
- if output_path:
55
- record_events(output_path=output_path, events=agents_thoughts)
48
+ def log_agent_thought(prefix: str, agent_thought: str):
49
+ if prefix:
50
+ prefix = prefix[0].upper() + prefix[1:]
51
+ else:
52
+ prefix = "New agent thought"
53
+ logger.info(
54
+ f"💭 {Fore.LIGHTMAGENTA_EX + Style.BRIGHT}{prefix}{Style.RESET_ALL}: "
55
+ f"{Fore.LIGHTMAGENTA_EX}{agent_thought}{Style.RESET_ALL}"
56
+ )
@@ -1,7 +1,45 @@
1
1
  from typing import Optional
2
2
 
3
+ from pydantic import BaseModel
3
4
 
4
- def find_element_by_resource_id(ui_hierarchy: list[dict], resource_id: str) -> Optional[dict]:
5
+ from minitap.mobile_use.utils.logger import get_logger
6
+
7
+ logger = get_logger(__name__)
8
+
9
+
10
+ def __find_element_by_ressource_id_in_rich_hierarchy(
11
+ hierarchy: list[dict], resource_id: str
12
+ ) -> Optional[dict]:
13
+ """
14
+ Retrieves all the sibling elements for a given resource ID from a nested dictionary.
15
+
16
+ Args:
17
+ hierarchy (dict): The nested dictionary representing the UI hierarchy.
18
+ resource_id (str): The resource-id to find.
19
+
20
+ Returns:
21
+ list: A list of the sibling elements, or None if the resource_id is not found.
22
+ """
23
+ if not hierarchy:
24
+ return None
25
+
26
+ for child in hierarchy:
27
+ if child.get("attributes", {}).get("resource-id") == resource_id:
28
+ return child.get("attributes", {})
29
+
30
+ for child in hierarchy:
31
+ result = __find_element_by_ressource_id_in_rich_hierarchy(
32
+ child.get("children", []), resource_id
33
+ )
34
+ if result is not None:
35
+ return result
36
+
37
+ return None
38
+
39
+
40
+ def find_element_by_resource_id(
41
+ ui_hierarchy: list[dict], resource_id: str, is_rich_hierarchy: bool = False
42
+ ) -> Optional[dict]:
5
43
  """
6
44
  Find a UI element by its resource-id in the UI hierarchy.
7
45
 
@@ -13,6 +51,8 @@ def find_element_by_resource_id(ui_hierarchy: list[dict], resource_id: str) -> O
13
51
  Returns:
14
52
  The complete UI element dictionary if found, None otherwise
15
53
  """
54
+ if is_rich_hierarchy:
55
+ return __find_element_by_ressource_id_in_rich_hierarchy(ui_hierarchy, resource_id)
16
56
 
17
57
  def search_recursive(elements: list[dict]) -> Optional[dict]:
18
58
  for element in elements:
@@ -28,3 +68,50 @@ def find_element_by_resource_id(ui_hierarchy: list[dict], resource_id: str) -> O
28
68
  return None
29
69
 
30
70
  return search_recursive(ui_hierarchy)
71
+
72
+
73
+ def is_element_focused(element: dict) -> bool:
74
+ return element.get("focused", None) == "true"
75
+
76
+
77
+ class Point(BaseModel):
78
+ x: int
79
+ y: int
80
+
81
+
82
+ class ElementBounds(BaseModel):
83
+ x: int
84
+ y: int
85
+ width: int
86
+ height: int
87
+
88
+ def get_center(self) -> Point:
89
+ return Point(x=self.x + self.width // 2, y=self.y + self.height // 2)
90
+
91
+ def get_relative_point(self, x_percent: float, y_percent: float) -> Point:
92
+ """
93
+ Returns the coordinates of the point at x_percent of the width and y_percent
94
+ of the height of the element.
95
+
96
+ Ex if x_percent = 0.95 and y_percent = 0.95,
97
+ the point is at the bottom right of the element:
98
+ <------>
99
+ | |
100
+ | x|
101
+ <------>
102
+ """
103
+ return Point(
104
+ x=int((self.x + self.width) * x_percent),
105
+ y=int((self.y + self.height) * y_percent),
106
+ )
107
+
108
+
109
+ def get_bounds_for_element(element: dict) -> ElementBounds | None:
110
+ bounds = element.get("bounds", None)
111
+ if bounds:
112
+ try:
113
+ return ElementBounds(**bounds)
114
+ except Exception as e:
115
+ logger.error(f"Failed to validate bounds: {e}")
116
+ return None
117
+ return None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: minitap-mobile-use
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: AI-powered multi-agent system that automates real Android and iOS devices through low-level control using LangGraph.
5
5
  Author: Pierre-Louis Favreau, Jean-Pierre Lo, Nicolas Dehandschoewercker
6
6
  License: MIT License
@@ -45,7 +45,7 @@ Requires-Dist: psutil>=5.9.0
45
45
  Requires-Dist: ruff==0.5.3 ; extra == 'dev'
46
46
  Requires-Dist: pytest==8.4.1 ; extra == 'dev'
47
47
  Requires-Dist: pytest-cov==5.0.0 ; extra == 'dev'
48
- Requires-Python: >=3.10
48
+ Requires-Python: >=3.12
49
49
  Project-URL: Homepage, https://minitap.ai/
50
50
  Project-URL: Source, https://github.com/minitap-ai/mobile-use
51
51
  Provides-Extra: dev
@@ -0,0 +1,94 @@
1
+ minitap/mobile_use/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
2
+ minitap/mobile_use/agents/contextor/contextor.py,sha256=d7c98771a9173281c1660a2ee965149d4f70ecf7005cf75fa89eb1e9a9f2b660,1673
3
+ minitap/mobile_use/agents/cortex/cortex.md,sha256=e844462b17614ca1f1908221b07d2c3896fb19d863c18ecdde8f70a0e2f9c962,5273
4
+ minitap/mobile_use/agents/cortex/cortex.py,sha256=b6922e920971e5f5a7f03c92e4f17665486c067d845f53973a6ccac6b1f4f9b7,4671
5
+ minitap/mobile_use/agents/cortex/types.py,sha256=de700f65f212ba2f2a9a004d353c05723ee5fdf3b2541b1882668fabf83d3ad9,393
6
+ minitap/mobile_use/agents/executor/executor.md,sha256=a20682e2936b6096cee82ccbc51370c20eb35f332d6994620b584749c953c339,2981
7
+ minitap/mobile_use/agents/executor/executor.py,sha256=be92742d91296650288c12e83f8065589d8eab7aeaa977a47c34ed25879a9bb6,2771
8
+ minitap/mobile_use/agents/executor/tool_node.py,sha256=6971224081589202ecdcb2592dcbd31155ccfb4266f961f4c56cda23221a9d70,3933
9
+ minitap/mobile_use/agents/executor/utils.py,sha256=1a387d30047d3be05b5e550433866abe4388222b3f95d1360847870155ef8f12,368
10
+ minitap/mobile_use/agents/hopper/hopper.md,sha256=2e9333ece8f6b76401ac2cce98ca06a025faa5dba6bacbbc344793ddf42292d0,362
11
+ minitap/mobile_use/agents/hopper/hopper.py,sha256=1c015cc911f225c9cb26415e0852d5905ded8e715fb928ad2595143870a781e3,1324
12
+ minitap/mobile_use/agents/orchestrator/human.md,sha256=6559026aa921b7ad7dddcf3dfcd5d9930252edd6484d60ea92ff6ca97ed028fc,229
13
+ minitap/mobile_use/agents/orchestrator/orchestrator.md,sha256=cc1a353c577f2eef42d9a528178ccc2c0a4a144a907a29878f9efe57e83b12fa,2546
14
+ minitap/mobile_use/agents/orchestrator/orchestrator.py,sha256=83524eefe8fc1d9639a2ffeb270dfed9e2a2b15af93e6e1d3d41d8e7c5b74951,4656
15
+ minitap/mobile_use/agents/orchestrator/types.py,sha256=f53dfdc99e8d50888ac1cde5f7f90ba5c87837a8eee8dd8efa31f2640394433c,335
16
+ minitap/mobile_use/agents/outputter/human.md,sha256=6b9b45c640b163554524b1aec4cd97134c628eeb8557a32e23c8f966d32f642e,771
17
+ minitap/mobile_use/agents/outputter/outputter.py,sha256=fd6143776e916439f39d823bf6860e3138d1d191b89d60d8cc809fbb49bd3808,2792
18
+ minitap/mobile_use/agents/outputter/test_outputter.py,sha256=4c52988f8f29159657707a0e11500610f3987cac6390b17edec23f09ddfcc0ff,3334
19
+ minitap/mobile_use/agents/planner/human.md,sha256=cb37be2af568918e60238eaa785837178a3ba8f8112de86850d9a62914c18314,222
20
+ minitap/mobile_use/agents/planner/planner.md,sha256=db777bb2225625528470565cfed539b115b96c8304dbfc1810d9afca78b43e10,2901
21
+ minitap/mobile_use/agents/planner/planner.py,sha256=2450e5479dfa19c8bb9c5354def952d35cb6a379d7a4a40880127c1186fe994f,2513
22
+ minitap/mobile_use/agents/planner/types.py,sha256=f8da73b57efd89dc60bbe9cade31492e5179f1c0aca8d6e7d296c32e2275e2ad,1437
23
+ minitap/mobile_use/agents/planner/utils.py,sha256=11731d0382bf88ef77b38aaf75873486f293343cdd048372c57e626c652b4a22,1839
24
+ minitap/mobile_use/agents/summarizer/summarizer.py,sha256=63899afe2c9c7ecbba4d1d6fdf18c865a2310cfe76e4c6c9b31e74d6746eb5b7,1070
25
+ minitap/mobile_use/clients/device_hardware_client.py,sha256=9593380a7a3df32f02aa22717678c25e91367df26b1743abde9e57aec5dc2474,857
26
+ minitap/mobile_use/clients/ios_client.py,sha256=332bf47ac01bbd5bf6178a59eea7c7a30fed944f30907caea5388178f312d36b,1452
27
+ minitap/mobile_use/clients/screen_api_client.py,sha256=3615dc65d25c38b4d8dc5512f9adb3bcf69dca7a0298a472a6812f604a275c47,2019
28
+ minitap/mobile_use/config.py,sha256=8d03a1b0cf06ff0a311671160d0e9fab07a0c3f2f9bab1f0a5f3352ea8b4eec4,9470
29
+ minitap/mobile_use/constants.py,sha256=3acd9d6ade5bc772e902b3473f3ba12ddd04e7306963ca2bae49d1132d89ba46,95
30
+ minitap/mobile_use/context.py,sha256=8bed4b88c01c84a3686aebaa3b66454cc705f000d694ecd2ea73753be29b0c01,1792
31
+ minitap/mobile_use/controllers/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
32
+ minitap/mobile_use/controllers/mobile_command_controller.py,sha256=d5059dc346ab19d4aefb14a33b85f4459332f39e757a1a208770b519ec993c2f,11767
33
+ minitap/mobile_use/controllers/platform_specific_commands_controller.py,sha256=3d62496216bd6e16a7127fda866a1eddd9086c6d5d19a23a997dc44f56d6e8f4,2745
34
+ minitap/mobile_use/graph/graph.py,sha256=76d3c9e30f7f3581d225f491b917e94a7af52855c8541b1764eea07ad925e287,4268
35
+ minitap/mobile_use/graph/state.py,sha256=6c77b562d0839acc4f614b045542930ca0777b7faf957a553933a98ab3212b35,3416
36
+ minitap/mobile_use/main.py,sha256=5183231fcc9dba33a6a047395845d6ed88fe3973476328e79f05ece8dfc20b06,3794
37
+ minitap/mobile_use/sdk/__init__.py,sha256=4e5555c0597242b9523827194a2500b9c6d7e5c04b1ccd2056c9b1f4d42a31cd,318
38
+ minitap/mobile_use/sdk/agent.py,sha256=37d2a889e9637c944085a23bc65ac3f4d06eafd97a86bd7f1124907546d0c1c5,20977
39
+ minitap/mobile_use/sdk/builders/__init__.py,sha256=d6c96d39b80900a114698ef205ab5061a541f33bfa99c456d9345e5adb8ff6ff,424
40
+ minitap/mobile_use/sdk/builders/agent_config_builder.py,sha256=752645967f68a1748928d9c4915bba8e38bdf96fa2fbe53417c643934458fac0,7655
41
+ minitap/mobile_use/sdk/builders/index.py,sha256=64336ac3b3dea4673a48e95b8c5ac4196ecd5d2196380377d102593d0a1dc138,442
42
+ minitap/mobile_use/sdk/builders/task_request_builder.py,sha256=739f7886cbcc87602f83f5fa37f5da049323a7754f2254840bedc1f74abe19d2,6906
43
+ minitap/mobile_use/sdk/constants.py,sha256=436ba0700c6cf37ac0c9e3995a5f5a0d54ca87af72686eb9667a2c6a96e30f68,292
44
+ minitap/mobile_use/sdk/examples/README.md,sha256=b6780de3025b1897ad2ef42fa968f2a69b82f929465807f9590c4e9321dab6e6,2306
45
+ minitap/mobile_use/sdk/examples/__init__.py,sha256=c23868a2ca7e9b76e80d6835fe93c10e13ea8f2287dd6e785511b8ac30354e9b,46
46
+ minitap/mobile_use/sdk/examples/simple_photo_organizer.py,sha256=8ad1cebb5281e3663264560bd15b090add41d2821b1db77e4cbc829860c98df8,2606
47
+ minitap/mobile_use/sdk/examples/smart_notification_assistant.py,sha256=bf43241c33b882069d5845a6f0ea78038e4c484299893ce8345cebc5836a4e0c,6386
48
+ minitap/mobile_use/sdk/types/__init__.py,sha256=5dd148d83bf6261ac8ac60c994e2496b5bf535591d0835807d5fe394fd85a954,1014
49
+ minitap/mobile_use/sdk/types/agent.py,sha256=56b7de4e1cca882a2dd40eef9e837a951d70588b8b18013a88e2604c4114e45d,2285
50
+ minitap/mobile_use/sdk/types/exceptions.py,sha256=56ac3f749730740951448b1b0f200be21331dc0800916a87587b21e7850120a5,2288
51
+ minitap/mobile_use/sdk/types/task.py,sha256=d94e69dd0c58ecc8b613969dd223b45177a174b8a54f97f3a1cbf251517f31be,5915
52
+ minitap/mobile_use/sdk/utils.py,sha256=c32794fa7877062963c9fc035625668506b5ae05ae06aa47364356103fdce690,950
53
+ minitap/mobile_use/servers/config.py,sha256=b77296e56b8a0a3211bff4795d473cdc5fd58b31d3acaa8f9d920e6e4bed68f9,606
54
+ minitap/mobile_use/servers/device_hardware_bridge.py,sha256=ac446958157e7049dbf6eda6c8a55d710a28c221cb0042f271f63033186a2780,7211
55
+ minitap/mobile_use/servers/device_screen_api.py,sha256=63bf866f17cde4ab97631b710080866b8427225d3857b2351ab83db38a9c5107,5064
56
+ minitap/mobile_use/servers/start_servers.py,sha256=04fd57f629c39bfa012af6d953280567c7b9eb1fa1ad055b5d1f3c792a9b22f5,5056
57
+ minitap/mobile_use/servers/stop_servers.py,sha256=488e93ce0d6c97f42b91c371f7fc1848977e609c38a99b6d0247684587f883cd,7151
58
+ minitap/mobile_use/servers/utils.py,sha256=db5d26153a169ab141556337db3693adc1bf8522943316656bdeb05dbf95465b,394
59
+ minitap/mobile_use/services/accessibility.py,sha256=42bcbe81b427ee6f6e82bcfe420fc40630db950bda354e3e433c2dda2e159628,3404
60
+ minitap/mobile_use/services/llm.py,sha256=69a413fe68f98ded37d8f04e5ffe53805db8f0f96b94763698f268f665ea75bb,3708
61
+ minitap/mobile_use/tools/index.py,sha256=40d00ec58961f25140a53159a99619be2dbb7b8058f4a75318017ad1c2eb34ee,2170
62
+ minitap/mobile_use/tools/mobile/back.py,sha256=9e6cd050427959a9c1a3a7592819f6c32c4dbb22751e64dfc74d46ed56b6b453,1803
63
+ minitap/mobile_use/tools/mobile/copy_text_from.py,sha256=f6a8434e9fec81cd311c482340fe3591fc2f0cf1b0a2eef8f368754aea677db6,2727
64
+ minitap/mobile_use/tools/mobile/erase_text.py,sha256=a179302928020dd0b81fea821909bc4ad7a5842ca0f09e0bb1219b816ab0b234,4513
65
+ minitap/mobile_use/tools/mobile/find_packages.py,sha256=fadce1846d3b0bee4eeb75179bdfdec6c80e3051e6298f3789ff40fd4c4d79c5,2565
66
+ minitap/mobile_use/tools/mobile/input_text.py,sha256=2afd159d97c13e1276479452e9517f67d957777c17bb024b2ae6479b828d18c9,5528
67
+ minitap/mobile_use/tools/mobile/launch_app.py,sha256=1241be67e90c2f7fd815be29b541ba709a26d6de6d5c9c23f7ca9b007ca6f872,2038
68
+ minitap/mobile_use/tools/mobile/long_press_on.py,sha256=cfd8484160084f73f28689748bd22de0e710af1484ff19f97b6f4fd98a580957,2237
69
+ minitap/mobile_use/tools/mobile/open_link.py,sha256=d112507692d45bf460d205c5a1340b8ef19b7182174d9477faf3624b8b52fe18,1886
70
+ minitap/mobile_use/tools/mobile/paste_text.py,sha256=220b9d8820a8726b42667b6d6bee04e2f33a99464cc3b857b4bc1f5bdcaddb1b,2112
71
+ minitap/mobile_use/tools/mobile/press_key.py,sha256=82c43ae2e0a519b974643009daba960226c78a97f328f1fbaa6ed15b5d8561ee,1950
72
+ minitap/mobile_use/tools/mobile/stop_app.py,sha256=9251f83591e97a20b7bdcb8cd9c64bbcc8c1f0e1295719a76ff129f0bbaf2cc1,2105
73
+ minitap/mobile_use/tools/mobile/swipe.py,sha256=d545c948cb9c98758f44287f19f44f229b4fb6b50152a30938704a3c0fbe460b,1889
74
+ minitap/mobile_use/tools/mobile/take_screenshot.py,sha256=0708f6fbfe600cbf42e844602b439b03f6f6c5608e8c498089b150bc6946f216,2282
75
+ minitap/mobile_use/tools/mobile/tap.py,sha256=8a6c38ede8a0ae6f69abbcc9e408b00ccc82efa96f6e44a4a08348800e1eaa9f,2340
76
+ minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py,sha256=bbcd01c54473f446694a1fa4213a9653c86cd84cffce698e2915e391998cc6ab,2536
77
+ minitap/mobile_use/tools/tool_wrapper.py,sha256=fe5042803877973ec4ddbdc235d4d3e4bb64e1b2e0987ef58a7bc7112cb29b98,325
78
+ minitap/mobile_use/utils/cli_helpers.py,sha256=00578e45bfa71a6b332026572bbf255716f40f0499dd8a246f075f801bce7c7d,1751
79
+ minitap/mobile_use/utils/cli_selection.py,sha256=8cbdb9559f4f3d40ae3ad5287239050cfc7ca826ecf2cb488c8c6d255dab0cd3,4828
80
+ minitap/mobile_use/utils/conversations.py,sha256=8f1d924300ec3f6f7c71510c21e3011b75caca5b1fff06fdaccb377c3cde24ec,914
81
+ minitap/mobile_use/utils/decorators.py,sha256=c88b4ab78724cb97d4c8d40c6049e874293e8d0c0ee6cb0131aa869d9b205810,3631
82
+ minitap/mobile_use/utils/errors.py,sha256=6c5566484cff48ce1eb168c3cbe93d6e5365457828f59616a15147598feef769,156
83
+ minitap/mobile_use/utils/file.py,sha256=1ca968613452a273b23e4f58460ab39f87255b02cdb6fb8ca04f4e628b346070,315
84
+ minitap/mobile_use/utils/logger.py,sha256=ff75d4e373cc750ba4cdaee6c86139841420c44e8ad291f7f3afc77367dc7daa,5695
85
+ minitap/mobile_use/utils/media.py,sha256=cd51b2f1c268aca2dd56439a97d761765454a92e9598d76c0a86f62185b990f9,2233
86
+ minitap/mobile_use/utils/recorder.py,sha256=83dcb953b9caf00bf3e4853c3eee447166e1279e7ceefbbeb25d93e932ec698d,1937
87
+ minitap/mobile_use/utils/requests_utils.py,sha256=5c3a2e2aff7c521cd6a43b74c084e2244e1ff55065a5b722e6e251c6419861fd,1168
88
+ minitap/mobile_use/utils/shell_utils.py,sha256=b35ae7f863379adb86c9ba0f9b3b9d4954118d12aef1ffed0bc260b32d73d857,650
89
+ minitap/mobile_use/utils/time.py,sha256=41bfaabb3751de11443ccb4a3f1f53d5ebacc7744c72e32695fdcc3d23f17d49,160
90
+ minitap/mobile_use/utils/ui_hierarchy.py,sha256=86407313d163fc36f1c2c486f91d74997119d56d3da3b0f1329de3ad4f8954fe,3373
91
+ minitap_mobile_use-2.0.1.dist-info/WHEEL,sha256=ab6157bc637547491fb4567cd7ddf26b04d63382916ca16c29a5c8e94c9c9ef7,79
92
+ minitap_mobile_use-2.0.1.dist-info/entry_points.txt,sha256=663a29cfd551a4eaa0f27335f0bd7e4a732a4e39c76b68ef5c8dc444d4a285fa,60
93
+ minitap_mobile_use-2.0.1.dist-info/METADATA,sha256=12bfd3e4cfa4719cb84b225a28544f597fe78cdd4a2c1694f04a6f7347b63093,10363
94
+ minitap_mobile_use-2.0.1.dist-info/RECORD,,