minitap-mobile-use 2.2.0__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of minitap-mobile-use might be problematic. Click here for more details.

Files changed (59) hide show
  1. minitap/mobile_use/agents/contextor/contextor.py +6 -4
  2. minitap/mobile_use/agents/cortex/cortex.md +114 -27
  3. minitap/mobile_use/agents/cortex/cortex.py +8 -5
  4. minitap/mobile_use/agents/executor/executor.md +15 -10
  5. minitap/mobile_use/agents/executor/executor.py +6 -5
  6. minitap/mobile_use/agents/executor/utils.py +2 -1
  7. minitap/mobile_use/agents/hopper/hopper.py +6 -3
  8. minitap/mobile_use/agents/orchestrator/orchestrator.py +26 -11
  9. minitap/mobile_use/agents/outputter/outputter.py +6 -3
  10. minitap/mobile_use/agents/outputter/test_outputter.py +104 -42
  11. minitap/mobile_use/agents/planner/planner.md +20 -22
  12. minitap/mobile_use/agents/planner/planner.py +10 -7
  13. minitap/mobile_use/agents/planner/types.py +4 -2
  14. minitap/mobile_use/agents/planner/utils.py +14 -0
  15. minitap/mobile_use/agents/summarizer/summarizer.py +2 -2
  16. minitap/mobile_use/config.py +6 -1
  17. minitap/mobile_use/context.py +13 -3
  18. minitap/mobile_use/controllers/mobile_command_controller.py +1 -14
  19. minitap/mobile_use/graph/state.py +7 -3
  20. minitap/mobile_use/sdk/agent.py +204 -29
  21. minitap/mobile_use/sdk/examples/README.md +19 -1
  22. minitap/mobile_use/sdk/examples/platform_minimal_example.py +46 -0
  23. minitap/mobile_use/sdk/services/platform.py +244 -0
  24. minitap/mobile_use/sdk/types/__init__.py +14 -14
  25. minitap/mobile_use/sdk/types/exceptions.py +57 -0
  26. minitap/mobile_use/sdk/types/platform.py +125 -0
  27. minitap/mobile_use/sdk/types/task.py +60 -17
  28. minitap/mobile_use/servers/device_hardware_bridge.py +3 -2
  29. minitap/mobile_use/servers/stop_servers.py +11 -12
  30. minitap/mobile_use/servers/utils.py +6 -9
  31. minitap/mobile_use/services/llm.py +89 -5
  32. minitap/mobile_use/tools/index.py +2 -8
  33. minitap/mobile_use/tools/mobile/back.py +3 -3
  34. minitap/mobile_use/tools/mobile/clear_text.py +67 -38
  35. minitap/mobile_use/tools/mobile/erase_one_char.py +5 -4
  36. minitap/mobile_use/tools/mobile/{take_screenshot.py → glimpse_screen.py} +23 -15
  37. minitap/mobile_use/tools/mobile/input_text.py +67 -16
  38. minitap/mobile_use/tools/mobile/launch_app.py +54 -22
  39. minitap/mobile_use/tools/mobile/long_press_on.py +15 -8
  40. minitap/mobile_use/tools/mobile/open_link.py +15 -8
  41. minitap/mobile_use/tools/mobile/press_key.py +15 -8
  42. minitap/mobile_use/tools/mobile/stop_app.py +14 -8
  43. minitap/mobile_use/tools/mobile/swipe.py +11 -5
  44. minitap/mobile_use/tools/mobile/tap.py +103 -21
  45. minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py +3 -3
  46. minitap/mobile_use/tools/test_utils.py +377 -0
  47. minitap/mobile_use/tools/types.py +35 -0
  48. minitap/mobile_use/tools/utils.py +149 -39
  49. minitap/mobile_use/utils/recorder.py +1 -1
  50. minitap/mobile_use/utils/test_ui_hierarchy.py +178 -0
  51. minitap/mobile_use/utils/ui_hierarchy.py +11 -4
  52. {minitap_mobile_use-2.2.0.dist-info → minitap_mobile_use-2.4.0.dist-info}/METADATA +6 -4
  53. minitap_mobile_use-2.4.0.dist-info/RECORD +99 -0
  54. minitap/mobile_use/tools/mobile/copy_text_from.py +0 -73
  55. minitap/mobile_use/tools/mobile/find_packages.py +0 -69
  56. minitap/mobile_use/tools/mobile/paste_text.py +0 -62
  57. minitap_mobile_use-2.2.0.dist-info/RECORD +0 -96
  58. {minitap_mobile_use-2.2.0.dist-info → minitap_mobile_use-2.4.0.dist-info}/WHEEL +0 -0
  59. {minitap_mobile_use-2.2.0.dist-info → minitap_mobile_use-2.4.0.dist-info}/entry_points.txt +0 -0
@@ -8,40 +8,56 @@ from minitap.mobile_use.controllers.mobile_command_controller import (
8
8
  tap,
9
9
  )
10
10
  from minitap.mobile_use.graph.state import State
11
+ from minitap.mobile_use.tools.types import Target
11
12
  from minitap.mobile_use.utils.logger import get_logger
12
13
  from minitap.mobile_use.utils.ui_hierarchy import (
14
+ ElementBounds,
13
15
  Point,
14
16
  find_element_by_resource_id,
15
17
  get_bounds_for_element,
18
+ get_element_text,
16
19
  is_element_focused,
17
20
  )
18
21
 
19
22
  logger = get_logger(__name__)
20
23
 
21
24
 
22
- def move_cursor_to_end_if_bounds(
23
- ctx: MobileUseContext,
24
- state: State,
25
- resource_id: str,
26
- elt: dict | None = None,
25
+ def find_element_by_text(
26
+ ui_hierarchy: list[dict], text: str, index: int | None = None
27
27
  ) -> dict | None:
28
28
  """
29
- Best-effort move of the text cursor near the end of the input by tapping the
30
- bottom-right area of the focused element (if bounds are available).
29
+ Find a UI element by its text content (adapted to both flat and rich hierarchy)
30
+
31
+ This function performs a recursive, case-insensitive partial search.
32
+
33
+ Args:
34
+ ui_hierarchy: List of UI element dictionaries.
35
+ text: The text content to search for.
36
+
37
+ Returns:
38
+ The complete UI element dictionary if found, None otherwise.
31
39
  """
32
- if not elt:
33
- elt = find_element_by_resource_id(
34
- ui_hierarchy=state.latest_ui_hierarchy or [],
35
- resource_id=resource_id,
36
- )
37
- if not elt:
38
- return
39
40
 
40
- bounds = get_bounds_for_element(elt)
41
- if not bounds:
42
- return elt
41
+ def search_recursive(elements: list[dict]) -> dict | None:
42
+ for element in elements:
43
+ if isinstance(element, dict):
44
+ src = element.get("attributes", element)
45
+ if text and text.lower() == src.get("text", "").lower():
46
+ idx = index or 0
47
+ if idx == 0:
48
+ return element
49
+ idx -= 1
50
+ continue
51
+ if (children := element.get("children", [])) and (
52
+ found := search_recursive(children)
53
+ ):
54
+ return found
55
+ return None
56
+
57
+ return search_recursive(ui_hierarchy)
58
+
43
59
 
44
- logger.debug("Tapping near the end of the input to move the cursor")
60
+ def tap_bottom_right_of_element(bounds: ElementBounds, ctx: MobileUseContext):
45
61
  bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
46
62
  tap(
47
63
  ctx=ctx,
@@ -52,35 +68,129 @@ def move_cursor_to_end_if_bounds(
52
68
  ),
53
69
  ),
54
70
  )
55
- logger.debug(f"Tapped end of input {resource_id} at ({bottom_right.x}, {bottom_right.y})")
56
- return elt
57
71
 
58
72
 
59
- def focus_element_if_needed(
73
+ def move_cursor_to_end_if_bounds(
60
74
  ctx: MobileUseContext,
61
- resource_id: str,
62
- ) -> bool:
75
+ state: State,
76
+ target: Target,
77
+ elt: dict | None = None,
78
+ ) -> dict | None:
63
79
  """
64
- Ensures the element identified by `resource_id` is focused.
80
+ Best-effort move of the text cursor near the end of the input by tapping the
81
+ bottom-right area of the focused element (if bounds are available).
65
82
  """
66
- rich_hierarchy: list[dict] = ctx.hw_bridge_client.get_rich_hierarchy()
67
- rich_elt = find_element_by_resource_id(
68
- ui_hierarchy=rich_hierarchy,
69
- resource_id=resource_id,
70
- is_rich_hierarchy=True,
71
- )
72
- if rich_elt and not is_element_focused(rich_elt):
73
- tap(ctx=ctx, selector_request=IdSelectorRequest(id=resource_id))
74
- logger.debug(f"Focused (tap) on resource_id={resource_id}")
75
- rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
76
- rich_elt = find_element_by_resource_id(
83
+ if target.resource_id:
84
+ if not elt:
85
+ elt = find_element_by_resource_id(
86
+ ui_hierarchy=state.latest_ui_hierarchy or [],
87
+ resource_id=target.resource_id,
88
+ index=target.resource_id_index,
89
+ )
90
+ if not elt:
91
+ return None
92
+
93
+ bounds = get_bounds_for_element(elt)
94
+ if not bounds:
95
+ return elt
96
+
97
+ logger.debug("Tapping near the end of the input to move the cursor")
98
+ tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
99
+ logger.debug(f"Tapped end of input {target.resource_id}")
100
+ return elt
101
+
102
+ if target.coordinates:
103
+ tap_bottom_right_of_element(target.coordinates, ctx=ctx)
104
+ logger.debug("Tapped end of input by coordinates")
105
+ return elt
106
+
107
+ if target.text:
108
+ text_elt = find_element_by_text(
109
+ state.latest_ui_hierarchy or [], target.text, index=target.text_index
110
+ )
111
+ if text_elt:
112
+ bounds = get_bounds_for_element(text_elt)
113
+ if bounds:
114
+ tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
115
+ logger.debug(f"Tapped end of input that had text'{target.text}'")
116
+ return text_elt
117
+ return None
118
+
119
+ return None
120
+
121
+
122
+ def focus_element_if_needed(ctx: MobileUseContext, target: Target) -> bool:
123
+ """
124
+ Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
125
+ """
126
+ rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
127
+ elt_from_id = None
128
+ if target.resource_id:
129
+ elt_from_id = find_element_by_resource_id(
77
130
  ui_hierarchy=rich_hierarchy,
78
- resource_id=resource_id,
131
+ resource_id=target.resource_id,
132
+ index=target.resource_id_index,
79
133
  is_rich_hierarchy=True,
80
134
  )
81
- if rich_elt and is_element_focused(rich_elt):
82
- logger.debug(f"Text input is focused: {resource_id}")
135
+
136
+ if elt_from_id and target.text:
137
+ text_from_id_elt = get_element_text(elt_from_id)
138
+ if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
139
+ logger.warning(
140
+ f"ID '{target.resource_id}' and text '{target.text}' seem to be on different "
141
+ "elements. Ignoring the resource_id and falling back to other locators."
142
+ )
143
+ elt_from_id = None
144
+
145
+ if elt_from_id:
146
+ if not is_element_focused(elt_from_id):
147
+ tap(
148
+ ctx=ctx,
149
+ selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore
150
+ index=target.resource_id_index,
151
+ )
152
+ logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
153
+ rich_hierarchy = ctx.hw_bridge_client.get_rich_hierarchy()
154
+ elt_from_id = find_element_by_resource_id(
155
+ ui_hierarchy=rich_hierarchy,
156
+ resource_id=target.resource_id, # type: ignore
157
+ index=target.resource_id_index,
158
+ is_rich_hierarchy=True,
159
+ )
160
+ if elt_from_id and is_element_focused(elt_from_id):
161
+ logger.debug(f"Text input is focused: {target.resource_id}")
162
+ return True
163
+ logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
164
+
165
+ if target.coordinates:
166
+ relative_point = target.coordinates.get_center()
167
+ tap(
168
+ ctx=ctx,
169
+ selector_request=SelectorRequestWithCoordinates(
170
+ coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
171
+ ),
172
+ )
173
+ logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
83
174
  return True
84
175
 
85
- logger.warning(f"Failed to focus resource_id={resource_id}")
176
+ if target.text:
177
+ text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
178
+ if text_elt:
179
+ bounds = get_bounds_for_element(text_elt)
180
+ if bounds:
181
+ relative_point = bounds.get_center()
182
+ tap(
183
+ ctx=ctx,
184
+ selector_request=SelectorRequestWithCoordinates(
185
+ coordinates=CoordinatesSelectorRequest(
186
+ x=relative_point.x, y=relative_point.y
187
+ )
188
+ ),
189
+ )
190
+ logger.debug(f"Tapped on text element '{target.text}' to focus.")
191
+ return True
192
+
193
+ logger.error(
194
+ "Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
195
+ )
86
196
  return False
@@ -25,7 +25,7 @@ def record_interaction(ctx: MobileUseContext, response: BaseMessage):
25
25
  logger.error(f"Error compressing screenshot: {e}")
26
26
  return "Could not record this interaction"
27
27
  timestamp = time.time()
28
- folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_id).resolve()
28
+ folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve()
29
29
  folder.mkdir(parents=True, exist_ok=True)
30
30
  try:
31
31
  with open(
@@ -0,0 +1,178 @@
1
+ from unittest.mock import patch
2
+
3
+ from minitap.mobile_use.utils.ui_hierarchy import (
4
+ ElementBounds,
5
+ Point,
6
+ find_element_by_resource_id,
7
+ get_bounds_for_element,
8
+ get_element_text,
9
+ is_element_focused,
10
+ text_input_is_empty,
11
+ )
12
+
13
+
14
+ def test_text_input_is_empty():
15
+ assert text_input_is_empty(text=None, hint_text=None)
16
+ assert text_input_is_empty(text="", hint_text=None)
17
+ assert text_input_is_empty(text="", hint_text="")
18
+ assert text_input_is_empty(text="text", hint_text="text")
19
+
20
+ assert not text_input_is_empty(text="text", hint_text=None)
21
+ assert not text_input_is_empty(text="text", hint_text="")
22
+
23
+
24
+ def test_find_element_by_resource_id():
25
+ ui_hierarchy = [
26
+ {"resourceId": "com.example:id/button1", "text": "Button 1", "children": []},
27
+ {
28
+ "resourceId": "com.example:id/container",
29
+ "children": [
30
+ {
31
+ "resourceId": "com.example:id/nested_button",
32
+ "text": "Nested Button",
33
+ "children": [],
34
+ }
35
+ ],
36
+ },
37
+ ]
38
+
39
+ result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
40
+ assert result is not None
41
+ assert result["resourceId"] == "com.example:id/button1"
42
+ assert result["text"] == "Button 1"
43
+
44
+ result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
45
+ assert result is not None
46
+ assert result["resourceId"] == "com.example:id/nested_button"
47
+ assert result["text"] == "Nested Button"
48
+
49
+ result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
50
+ assert result is None
51
+
52
+ result = find_element_by_resource_id([], "com.example:id/button1")
53
+ assert result is None
54
+
55
+
56
+ def test_find_element_by_resource_id_rich_hierarchy():
57
+ rich_hierarchy = [
58
+ {"attributes": {"resource-id": "com.example:id/button1"}, "children": []},
59
+ {
60
+ "attributes": {"resource-id": "com.example:id/container"},
61
+ "children": [
62
+ {"attributes": {"resource-id": "com.example:id/nested_button"}, "children": []}
63
+ ],
64
+ },
65
+ ]
66
+
67
+ result = find_element_by_resource_id(
68
+ rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
69
+ )
70
+ assert result is not None
71
+ assert result["resource-id"] == "com.example:id/button1"
72
+
73
+ result = find_element_by_resource_id(
74
+ rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
75
+ )
76
+ assert result is not None
77
+ assert result["resource-id"] == "com.example:id/nested_button"
78
+
79
+ result = find_element_by_resource_id(
80
+ rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
81
+ )
82
+ assert result is None
83
+
84
+
85
+ def test_is_element_focused():
86
+ focused_element = {"focused": "true"}
87
+ assert is_element_focused(focused_element)
88
+
89
+ non_focused_element = {"focused": "false"}
90
+ assert not is_element_focused(non_focused_element)
91
+
92
+ no_focused_element = {"text": "some text"}
93
+ assert not is_element_focused(no_focused_element)
94
+
95
+ none_focused_element = {"focused": None}
96
+ assert not is_element_focused(none_focused_element)
97
+
98
+
99
+ def test_get_element_text():
100
+ element = {"text": "Button Text", "hintText": "Hint Text"}
101
+ assert get_element_text(element) == "Button Text"
102
+ assert get_element_text(element, hint_text=False) == "Button Text"
103
+ assert get_element_text(element, hint_text=True) == "Hint Text"
104
+
105
+ element_no_text = {"hintText": "Hint Text"}
106
+ assert get_element_text(element_no_text) is None
107
+ assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
108
+ element_no_hint = {"text": "Button Text"}
109
+ assert get_element_text(element_no_hint) == "Button Text"
110
+ assert get_element_text(element_no_hint, hint_text=True) is None
111
+
112
+ empty_element = {}
113
+ assert get_element_text(empty_element) is None
114
+ assert get_element_text(empty_element, hint_text=True) is None
115
+
116
+
117
+ def test_get_bounds_for_element():
118
+ element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
119
+ bounds = get_bounds_for_element(element_with_bounds)
120
+ assert bounds is not None
121
+ assert isinstance(bounds, ElementBounds)
122
+ assert bounds.x == 10
123
+ assert bounds.y == 20
124
+ assert bounds.width == 100
125
+ assert bounds.height == 50
126
+
127
+ element_no_bounds = {"text": "Button"}
128
+ bounds = get_bounds_for_element(element_no_bounds)
129
+ assert bounds is None
130
+
131
+ # Suppress logger output for the invalid bounds test case
132
+ with patch("minitap.mobile_use.utils.ui_hierarchy.logger.error"):
133
+ element_invalid_bounds = {
134
+ "bounds": {
135
+ "x": "invalid", # Should be int
136
+ "y": 20,
137
+ "width": 100,
138
+ "height": 50,
139
+ }
140
+ }
141
+ bounds = get_bounds_for_element(element_invalid_bounds)
142
+ assert bounds is None
143
+
144
+
145
+ def test_element_bounds():
146
+ bounds = ElementBounds(x=10, y=20, width=100, height=50)
147
+
148
+ center = bounds.get_center()
149
+ assert isinstance(center, Point)
150
+ assert center.x == 60
151
+ assert center.y == 45
152
+
153
+ center_point = bounds.get_relative_point(0.5, 0.5)
154
+ assert isinstance(center_point, Point)
155
+ assert center_point.x == 60
156
+ assert center_point.y == 45
157
+
158
+ top_left = bounds.get_relative_point(0.0, 0.0)
159
+ assert top_left.x == 10
160
+ assert top_left.y == 20
161
+
162
+ bottom_right = bounds.get_relative_point(1.0, 1.0)
163
+ assert bottom_right.x == 110
164
+ assert bottom_right.y == 70
165
+ custom_point = bounds.get_relative_point(0.95, 0.95)
166
+ assert custom_point.x == 105
167
+ assert custom_point.y == 67
168
+
169
+
170
+ if __name__ == "__main__":
171
+ test_text_input_is_empty()
172
+ test_find_element_by_resource_id()
173
+ test_find_element_by_resource_id_rich_hierarchy()
174
+ test_is_element_focused()
175
+ test_get_element_text()
176
+ test_get_bounds_for_element()
177
+ test_element_bounds()
178
+ print("All tests passed")
@@ -40,7 +40,10 @@ def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
40
40
 
41
41
 
42
42
  def find_element_by_resource_id(
43
- ui_hierarchy: list[dict], resource_id: str, is_rich_hierarchy: bool = False
43
+ ui_hierarchy: list[dict],
44
+ resource_id: str,
45
+ index: int | None = None,
46
+ is_rich_hierarchy: bool = False,
44
47
  ) -> dict | None:
45
48
  """
46
49
  Find a UI element by its resource-id in the UI hierarchy.
@@ -60,7 +63,11 @@ def find_element_by_resource_id(
60
63
  for element in elements:
61
64
  if isinstance(element, dict):
62
65
  if element.get("resourceId") == resource_id:
63
- return element
66
+ idx = index or 0
67
+ if idx == 0:
68
+ return element
69
+ idx -= 1
70
+ continue
64
71
 
65
72
  children = element.get("children", [])
66
73
  if children:
@@ -109,8 +116,8 @@ class ElementBounds(BaseModel):
109
116
  <------>
110
117
  """
111
118
  return Point(
112
- x=int((self.x + self.width) * x_percent),
113
- y=int((self.y + self.height) * y_percent),
119
+ x=int(self.x + self.width * x_percent),
120
+ y=int(self.y + self.height * y_percent),
114
121
  )
115
122
 
116
123
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: minitap-mobile-use
3
- Version: 2.2.0
3
+ Version: 2.4.0
4
4
  Summary: AI-powered multi-agent system that automates real Android and iOS devices through low-level control using LangGraph.
5
5
  Author: Pierre-Louis Favreau, Jean-Pierre Lo, Nicolas Dehandschoewercker
6
6
  License: MIT License
@@ -43,9 +43,11 @@ Requires-Dist: uvicorn[standard]==0.30.1
43
43
  Requires-Dist: colorama>=0.4.6
44
44
  Requires-Dist: psutil>=5.9.0
45
45
  Requires-Dist: langchain-google-vertexai>=2.0.28
46
+ Requires-Dist: httpx>=0.28.1
46
47
  Requires-Dist: ruff==0.5.3 ; extra == 'dev'
47
48
  Requires-Dist: pytest==8.4.1 ; extra == 'dev'
48
49
  Requires-Dist: pytest-cov==5.0.0 ; extra == 'dev'
50
+ Requires-Dist: pyright==1.1.405 ; extra == 'dev'
49
51
  Requires-Python: >=3.12
50
52
  Project-URL: Homepage, https://minitap.ai/
51
53
  Project-URL: Source, https://github.com/minitap-ai/mobile-use
@@ -56,7 +58,7 @@ Description-Content-Type: text/markdown
56
58
 
57
59
  <div align="center">
58
60
 
59
- ![mobile-use in Action](./doc/linkedin-demo-with-text.gif)
61
+ ![mobile-use in Action](./doc/banner-v2.png)
60
62
 
61
63
  </div>
62
64
 
@@ -83,7 +85,7 @@ Mobile-use is a powerful, open-source AI agent that controls your Android or IOS
83
85
  ## ✨ Features
84
86
 
85
87
  - 🗣️ **Natural Language Control**: Interact with your phone using your native language.
86
- - 📱 **UI-Aware Automation**: Intelligently navigates through app interfaces.
88
+ - 📱 **UI-Aware Automation**: Intelligently navigates through app interfaces (note: currently has limited effectiveness with games as they don't provide accessibility tree data).
87
89
  - 📊 **Data Scraping**: Extract information from any app and structure it into your desired format (e.g., JSON) using a natural language description.
88
90
  - 🔧 **Extensible & Customizable**: Easily configure different LLMs to power the agents that power mobile-use.
89
91
 
@@ -152,7 +154,7 @@ Then run in your terminal:
152
154
 
153
155
  ```bash
154
156
  chmod +x mobile-use.sh
155
- ./mobile-use.sh \
157
+ bash ./mobile-use.sh \
156
158
  "Open Gmail, find first 3 unread emails, and list their sender and subject line" \
157
159
  --output-description "A JSON list of objects, each with 'sender' and 'subject' keys"
158
160
  ```
@@ -0,0 +1,99 @@
1
+ minitap/mobile_use/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
2
+ minitap/mobile_use/agents/contextor/contextor.py,sha256=820c32e11fc0a622fffe4170d4d8ec376c6ff44cb5c20dbd8379a031b1961615,1717
3
+ minitap/mobile_use/agents/cortex/cortex.md,sha256=b1390b54d2e350204013daa7cb750eadb5054470d9a125240c892a9db445982e,12192
4
+ minitap/mobile_use/agents/cortex/cortex.py,sha256=15ec2eb1e5636b84db06c928b9854f07352f0e86a50244aec5dc2d0775d84e59,5077
5
+ minitap/mobile_use/agents/cortex/types.py,sha256=c33f2277752644d2185d84add03a493adaa530096d046d73366ab9121d99b946,361
6
+ minitap/mobile_use/agents/executor/executor.md,sha256=1054c84bbc52709325e7ac4bb2f5a400cb439fee164694e1109c0e373d30ed27,3726
7
+ minitap/mobile_use/agents/executor/executor.py,sha256=59fa952f25556dbea70dde656a09b62e1adba011ce7e67f587f373de73d2a5f9,2974
8
+ minitap/mobile_use/agents/executor/tool_node.py,sha256=2ad729ede393882460ae3d180ac1c0e1ab1688f40b2017220aad1b059f6485c5,3900
9
+ minitap/mobile_use/agents/executor/utils.py,sha256=74cf2287053cd4fc763835870e5ae029ca36f6cd0ca7d50678ad05d52ab265b7,368
10
+ minitap/mobile_use/agents/hopper/hopper.md,sha256=2e9333ece8f6b76401ac2cce98ca06a025faa5dba6bacbbc344793ddf42292d0,362
11
+ minitap/mobile_use/agents/hopper/hopper.py,sha256=d3426ebf244633d2216f20ee015616b25d68127674926d36696b91774d84f3ca,1426
12
+ minitap/mobile_use/agents/orchestrator/human.md,sha256=6559026aa921b7ad7dddcf3dfcd5d9930252edd6484d60ea92ff6ca97ed028fc,229
13
+ minitap/mobile_use/agents/orchestrator/orchestrator.md,sha256=cc1a353c577f2eef42d9a528178ccc2c0a4a144a907a29878f9efe57e83b12fa,2546
14
+ minitap/mobile_use/agents/orchestrator/orchestrator.py,sha256=56a06879332284edbf25699fcfbf234cfef03e45ffcf0172acda2a1049852fce,5088
15
+ minitap/mobile_use/agents/orchestrator/types.py,sha256=f53dfdc99e8d50888ac1cde5f7f90ba5c87837a8eee8dd8efa31f2640394433c,335
16
+ minitap/mobile_use/agents/outputter/human.md,sha256=6b9b45c640b163554524b1aec4cd97134c628eeb8557a32e23c8f966d32f642e,771
17
+ minitap/mobile_use/agents/outputter/outputter.py,sha256=cc3db524f0fefb463f1baa07aab13e3dfb4704994f173f8d193b5c45034a518e,2855
18
+ minitap/mobile_use/agents/outputter/test_outputter.py,sha256=907b517c486f82a384a364e0bd202c00e8e8082c138461f6eead0e25c2779ba9,5538
19
+ minitap/mobile_use/agents/planner/human.md,sha256=cb37be2af568918e60238eaa785837178a3ba8f8112de86850d9a62914c18314,222
20
+ minitap/mobile_use/agents/planner/planner.md,sha256=f2089809c01263834f0ff264153993809f4ad3c32088305a5c7d421c1d6f4732,3407
21
+ minitap/mobile_use/agents/planner/planner.py,sha256=ee3aafb1519419e16d0fdf02272e6cd6c31ff51dcd0b17131ec599fa866fcf1b,2931
22
+ minitap/mobile_use/agents/planner/types.py,sha256=e8acf3c2d1505286a138b6f7c3ef36f397d154953311d0875e3ef35152653e7f,1496
23
+ minitap/mobile_use/agents/planner/utils.py,sha256=88b4b039e09cea254615ff3d0bc8951c3717a68702742e913a0176ecfbcaf495,2315
24
+ minitap/mobile_use/agents/summarizer/summarizer.py,sha256=56c2c7d5d48f4ba045b1401538db78b2ddbd43280389ed4cbc58ecd1da0c7610,1083
25
+ minitap/mobile_use/clients/device_hardware_client.py,sha256=9593380a7a3df32f02aa22717678c25e91367df26b1743abde9e57aec5dc2474,857
26
+ minitap/mobile_use/clients/ios_client.py,sha256=332bf47ac01bbd5bf6178a59eea7c7a30fed944f30907caea5388178f312d36b,1452
27
+ minitap/mobile_use/clients/screen_api_client.py,sha256=3615dc65d25c38b4d8dc5512f9adb3bcf69dca7a0298a472a6812f604a275c47,2019
28
+ minitap/mobile_use/config.py,sha256=70a9519f0c4732870ee0bb7cdb1a7adc5dca85e380edce9e8cd5d5572709c7d9,10206
29
+ minitap/mobile_use/constants.py,sha256=3acd9d6ade5bc772e902b3473f3ba12ddd04e7306963ca2bae49d1132d89ba46,95
30
+ minitap/mobile_use/context.py,sha256=2e212588a8a7caf58460fead0cae195c6386b7301026bbb2d78fe56c263ce6c3,2131
31
+ minitap/mobile_use/controllers/__init__.py,sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,0
32
+ minitap/mobile_use/controllers/mobile_command_controller.py,sha256=cff7993e91eeae74b0cb063f820b7450344637037467de4c592711961cee77ea,11681
33
+ minitap/mobile_use/controllers/platform_specific_commands_controller.py,sha256=8b4fc30108c242da41fd998751dbfd5e6a69e2957a2dbbe5d6fc43d6b55f727e,2705
34
+ minitap/mobile_use/graph/graph.py,sha256=c7b412e725b096eca8f212d704c3faf91d77eea4131f1fae7af1ee39bc57cdae,4269
35
+ minitap/mobile_use/graph/state.py,sha256=8efeef0f4bb10c933e54fdf17882ae63907ca557e4f1ae56bced8b5ac0f9ed17,3503
36
+ minitap/mobile_use/main.py,sha256=7ac4dc592e3ce72bff602d67ba2f25b9b5e45e07a316e548d7c8e73735abf43d,3725
37
+ minitap/mobile_use/sdk/__init__.py,sha256=4e5555c0597242b9523827194a2500b9c6d7e5c04b1ccd2056c9b1f4d42a31cd,318
38
+ minitap/mobile_use/sdk/agent.py,sha256=ad30436d2de47804bd13c54e40c3c8709ff17765bbd2703e332e0959f885c4d3,27825
39
+ minitap/mobile_use/sdk/builders/__init__.py,sha256=d6c96d39b80900a114698ef205ab5061a541f33bfa99c456d9345e5adb8ff6ff,424
40
+ minitap/mobile_use/sdk/builders/agent_config_builder.py,sha256=c63f452350822daab86d29c44a333909b623fc7ff7bcbf74e5f9104b24630bf5,7582
41
+ minitap/mobile_use/sdk/builders/index.py,sha256=64336ac3b3dea4673a48e95b8c5ac4196ecd5d2196380377d102593d0a1dc138,442
42
+ minitap/mobile_use/sdk/builders/task_request_builder.py,sha256=9e6cf7afb68af986d6a81487179bb79d28f63047a068725d92996dbcbe753376,6857
43
+ minitap/mobile_use/sdk/constants.py,sha256=436ba0700c6cf37ac0c9e3995a5f5a0d54ca87af72686eb9667a2c6a96e30f68,292
44
+ minitap/mobile_use/sdk/examples/README.md,sha256=c92e57d6efc5bce9f3c8668b20d9ce80396aefb7a685636ee84de8c7b9c1403e,3247
45
+ minitap/mobile_use/sdk/examples/__init__.py,sha256=c23868a2ca7e9b76e80d6835fe93c10e13ea8f2287dd6e785511b8ac30354e9b,46
46
+ minitap/mobile_use/sdk/examples/platform_minimal_example.py,sha256=58e57771362ef606045e238fe96cdf91313f161f233c5167acef71bb05c065ed,1321
47
+ minitap/mobile_use/sdk/examples/simple_photo_organizer.py,sha256=8ad1cebb5281e3663264560bd15b090add41d2821b1db77e4cbc829860c98df8,2606
48
+ minitap/mobile_use/sdk/examples/smart_notification_assistant.py,sha256=1d00658dc30c7bce5ef68369f982cd2d1932f53e2e2da6ded70cea13dc669c72,6362
49
+ minitap/mobile_use/sdk/services/platform.py,sha256=20ea13966b9729f888fe0056487ebf897110fe25471f651eb3c31ed3aa93f498,9371
50
+ minitap/mobile_use/sdk/types/__init__.py,sha256=43a337d20f5c3088be3dc2699c4896f2eeb2db92745b6e47bb3e5cfb86fba762,1032
51
+ minitap/mobile_use/sdk/types/agent.py,sha256=390d5c642b3480f4a2203ddd28ec115c785f2576bec81e82e4db3c129399c020,2260
52
+ minitap/mobile_use/sdk/types/exceptions.py,sha256=684c0049c5af417edf7e46e515be14fd57a0614c81b06ed52f379bc9d0bbebf3,4499
53
+ minitap/mobile_use/sdk/types/platform.py,sha256=3c6fdb3d734f731c7f6d2ec4db64460a7696719423ad4ea0c13e00ba7b2acdac,5158
54
+ minitap/mobile_use/sdk/types/task.py,sha256=a375fb9203e22012001f9889f6b8d3f95a1f6edf911a31733108075bfbf30a18,7603
55
+ minitap/mobile_use/sdk/utils.py,sha256=647f1f4a463c3029c3b0eb3c33f7dd778d5f5fd9d293224f5474595a60e1de6f,967
56
+ minitap/mobile_use/servers/config.py,sha256=8a4a6bce23e2093d047a91e135e2f88627f76ac12177d071f25a3ca739b3afeb,575
57
+ minitap/mobile_use/servers/device_hardware_bridge.py,sha256=db5f55ba66bfebf70ea1a655d921d4070ca6c458b1e7e5e08ed58b3bea100d63,7227
58
+ minitap/mobile_use/servers/device_screen_api.py,sha256=63bf866f17cde4ab97631b710080866b8427225d3857b2351ab83db38a9c5107,5064
59
+ minitap/mobile_use/servers/start_servers.py,sha256=1e86dc0fcbdf6e6570ae68c7097145e754f3c3448ca813d415b3e5ebb74db828,5037
60
+ minitap/mobile_use/servers/stop_servers.py,sha256=04a409a17fc0323209301fe28fbb037d71e41e5422eb369640f9f329aae312f5,7064
61
+ minitap/mobile_use/servers/utils.py,sha256=f3cc85da39f8d60cb840001be418562de7db95462370db9b79e96d884abe5c17,294
62
+ minitap/mobile_use/services/accessibility.py,sha256=42bcbe81b427ee6f6e82bcfe420fc40630db950bda354e3e433c2dda2e159628,3404
63
+ minitap/mobile_use/services/llm.py,sha256=e100560940a5d4fc33d2086742369c621e81e6c2fbb98e6d07411856d4df480f,7059
64
+ minitap/mobile_use/tools/index.py,sha256=9eab29d43e66ca397fad3206a302a9a4f423448437ab0a6fdb2b9d8bb227513b,2377
65
+ minitap/mobile_use/tools/mobile/back.py,sha256=8b909e412c8ad382e339b8c89171bf1363398a95a5c96126f79ee2f24e3c2ed1,1816
66
+ minitap/mobile_use/tools/mobile/clear_text.py,sha256=58e0426f042e39d827d3ff2fe3a79bb23e4146cbefd13da938e1a6f397579c4b,9799
67
+ minitap/mobile_use/tools/mobile/erase_one_char.py,sha256=a125933619614b621479c514c8e29a7ec6b504b1e7ab1be13603a744f32322d6,1985
68
+ minitap/mobile_use/tools/mobile/glimpse_screen.py,sha256=c9ee978a1163cea4cd4900ea3fefc0e9fe2685578be47b63a2f925b3598fb51d,2528
69
+ minitap/mobile_use/tools/mobile/input_text.py,sha256=dd83d735bebcc0a0b376cd739a63e1db5c3031525090c262a3274e6f1aa97f7b,5427
70
+ minitap/mobile_use/tools/mobile/launch_app.py,sha256=a794665f9bc7a035ad6029b8153d6e56a0ba49ce36b0ab8a4e342267465d8fef,3237
71
+ minitap/mobile_use/tools/mobile/long_press_on.py,sha256=0a0263a1429207c36f5be5f3a6ae35be3e8c56897737ed9536ae83ffbdca43e6,2299
72
+ minitap/mobile_use/tools/mobile/open_link.py,sha256=3017b94a921ced032f18526101eeba8caa907ebee40b82bed0baa4f35477e6c8,1980
73
+ minitap/mobile_use/tools/mobile/press_key.py,sha256=0133b1f1eb3b67c94aeaa70ac3ff822dc1e1f70624a0727f6e8c3cbcef1a80ab,2044
74
+ minitap/mobile_use/tools/mobile/stop_app.py,sha256=74f93c60cdf039455897b0bb2d74e5702cc8429abc606b6069c7be6399b7fbdf,2166
75
+ minitap/mobile_use/tools/mobile/swipe.py,sha256=6436db271564ea211af3c6fd699b22f3205409533819a68d7193768321999a7d,5862
76
+ minitap/mobile_use/tools/mobile/tap.py,sha256=826fbc6471d641e145427f5963ca2c3a5bc46dceb9ed4d56053cc849825a54c1,5946
77
+ minitap/mobile_use/tools/mobile/wait_for_animation_to_end.py,sha256=26b422556a31700ca98e769e7142c02248980f2ce01b712440269a072d2c0efc,2517
78
+ minitap/mobile_use/tools/test_utils.py,sha256=42dd0ff789c92254bc4452ab339a53f0d597a9bfddd2a6554d37f54ff52920ef,13888
79
+ minitap/mobile_use/tools/tool_wrapper.py,sha256=f0f27beaae25a1bcfd9b72bf994de84b2e5fba9e242d8ad18a8d1a97cd7619e4,454
80
+ minitap/mobile_use/tools/types.py,sha256=4f73939260509269416f5ad515a34569f7fa3467e1f920b9e8a991d1adbc761a,1298
81
+ minitap/mobile_use/tools/utils.py,sha256=7a80b9f82447a718ecc1fd2163f13f35f4f0b4e2859abd85f3c014ded10bccf2,6926
82
+ minitap/mobile_use/utils/cli_helpers.py,sha256=1c53b6ea6cd2ba861302b182944c6a3a31dac27e316bca2c65cd6a3ca3256e81,1720
83
+ minitap/mobile_use/utils/cli_selection.py,sha256=62e949bf075e984b5d23b4a9880ff2bccf8f9e0f7ccb48120030a6a82075352b,4788
84
+ minitap/mobile_use/utils/conversations.py,sha256=8f1d924300ec3f6f7c71510c21e3011b75caca5b1fff06fdaccb377c3cde24ec,914
85
+ minitap/mobile_use/utils/decorators.py,sha256=0bb30fb4f5d5cef0aef45643e68e199d39910f1d771eb4086f3e083d566c16a5,3591
86
+ minitap/mobile_use/utils/errors.py,sha256=6c5566484cff48ce1eb168c3cbe93d6e5365457828f59616a15147598feef769,156
87
+ minitap/mobile_use/utils/file.py,sha256=1ca968613452a273b23e4f58460ab39f87255b02cdb6fb8ca04f4e628b346070,315
88
+ minitap/mobile_use/utils/logger.py,sha256=011fe08c39b111997ec685b9f0b378761607e35ac049234b5e86c2b58f29bbe3,5633
89
+ minitap/mobile_use/utils/media.py,sha256=29fa7f009a0f2bd60de2a9eba4a90e6eecc91cc67c2736b753506d48ab2bf5eb,2228
90
+ minitap/mobile_use/utils/recorder.py,sha256=a5f3e2da3b4326bd1285537b7cfd4c84eda3dc47cda2f3c873026d0720896823,1726
91
+ minitap/mobile_use/utils/requests_utils.py,sha256=5c3a2e2aff7c521cd6a43b74c084e2244e1ff55065a5b722e6e251c6419861fd,1168
92
+ minitap/mobile_use/utils/shell_utils.py,sha256=b35ae7f863379adb86c9ba0f9b3b9d4954118d12aef1ffed0bc260b32d73d857,650
93
+ minitap/mobile_use/utils/test_ui_hierarchy.py,sha256=96c1549c05b4f7254a22d57dbd40aea860756f1e0b9d8cc24319383643448422,5911
94
+ minitap/mobile_use/utils/time.py,sha256=41bfaabb3751de11443ccb4a3f1f53d5ebacc7744c72e32695fdcc3d23f17d49,160
95
+ minitap/mobile_use/utils/ui_hierarchy.py,sha256=f3370518035d9daf02c08042a9e28ad564f4fc81a2b268103b9a7f8bc5c61d11,3797
96
+ minitap_mobile_use-2.4.0.dist-info/WHEEL,sha256=ab6157bc637547491fb4567cd7ddf26b04d63382916ca16c29a5c8e94c9c9ef7,79
97
+ minitap_mobile_use-2.4.0.dist-info/entry_points.txt,sha256=663a29cfd551a4eaa0f27335f0bd7e4a732a4e39c76b68ef5c8dc444d4a285fa,60
98
+ minitap_mobile_use-2.4.0.dist-info/METADATA,sha256=1869bc355bd20880725667ac35b399d81b0512ee6d9770868fea35384ef193ee,11995
99
+ minitap_mobile_use-2.4.0.dist-info/RECORD,,