autoglm-gui 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. AutoGLM_GUI/__init__.py +11 -0
  2. AutoGLM_GUI/__main__.py +26 -8
  3. AutoGLM_GUI/actions/__init__.py +6 -0
  4. AutoGLM_GUI/actions/handler.py +196 -0
  5. AutoGLM_GUI/actions/types.py +15 -0
  6. AutoGLM_GUI/adb/__init__.py +53 -0
  7. AutoGLM_GUI/adb/apps.py +227 -0
  8. AutoGLM_GUI/adb/connection.py +323 -0
  9. AutoGLM_GUI/adb/device.py +171 -0
  10. AutoGLM_GUI/adb/input.py +67 -0
  11. AutoGLM_GUI/adb/screenshot.py +11 -0
  12. AutoGLM_GUI/adb/timing.py +167 -0
  13. AutoGLM_GUI/adb_plus/keyboard_installer.py +4 -2
  14. AutoGLM_GUI/adb_plus/qr_pair.py +8 -8
  15. AutoGLM_GUI/adb_plus/screenshot.py +22 -1
  16. AutoGLM_GUI/adb_plus/serial.py +38 -20
  17. AutoGLM_GUI/adb_plus/touch.py +4 -9
  18. AutoGLM_GUI/agents/__init__.py +51 -0
  19. AutoGLM_GUI/agents/events.py +19 -0
  20. AutoGLM_GUI/agents/factory.py +153 -0
  21. AutoGLM_GUI/agents/glm/__init__.py +7 -0
  22. AutoGLM_GUI/agents/glm/agent.py +292 -0
  23. AutoGLM_GUI/agents/glm/message_builder.py +81 -0
  24. AutoGLM_GUI/agents/glm/parser.py +110 -0
  25. AutoGLM_GUI/agents/glm/prompts_en.py +77 -0
  26. AutoGLM_GUI/agents/glm/prompts_zh.py +75 -0
  27. AutoGLM_GUI/agents/mai/__init__.py +28 -0
  28. AutoGLM_GUI/agents/mai/agent.py +405 -0
  29. AutoGLM_GUI/agents/mai/parser.py +254 -0
  30. AutoGLM_GUI/agents/mai/prompts.py +103 -0
  31. AutoGLM_GUI/agents/mai/traj_memory.py +91 -0
  32. AutoGLM_GUI/agents/protocols.py +27 -0
  33. AutoGLM_GUI/agents/stream_runner.py +188 -0
  34. AutoGLM_GUI/api/__init__.py +71 -11
  35. AutoGLM_GUI/api/agents.py +190 -229
  36. AutoGLM_GUI/api/control.py +9 -6
  37. AutoGLM_GUI/api/devices.py +112 -28
  38. AutoGLM_GUI/api/health.py +13 -0
  39. AutoGLM_GUI/api/history.py +78 -0
  40. AutoGLM_GUI/api/layered_agent.py +306 -181
  41. AutoGLM_GUI/api/mcp.py +11 -10
  42. AutoGLM_GUI/api/media.py +64 -1
  43. AutoGLM_GUI/api/scheduled_tasks.py +98 -0
  44. AutoGLM_GUI/api/version.py +23 -10
  45. AutoGLM_GUI/api/workflows.py +2 -1
  46. AutoGLM_GUI/config.py +72 -14
  47. AutoGLM_GUI/config_manager.py +98 -27
  48. AutoGLM_GUI/device_adapter.py +263 -0
  49. AutoGLM_GUI/device_manager.py +248 -29
  50. AutoGLM_GUI/device_protocol.py +266 -0
  51. AutoGLM_GUI/devices/__init__.py +49 -0
  52. AutoGLM_GUI/devices/adb_device.py +200 -0
  53. AutoGLM_GUI/devices/mock_device.py +185 -0
  54. AutoGLM_GUI/devices/remote_device.py +177 -0
  55. AutoGLM_GUI/exceptions.py +3 -3
  56. AutoGLM_GUI/history_manager.py +164 -0
  57. AutoGLM_GUI/i18n.py +81 -0
  58. AutoGLM_GUI/metrics.py +13 -20
  59. AutoGLM_GUI/model/__init__.py +5 -0
  60. AutoGLM_GUI/model/message_builder.py +69 -0
  61. AutoGLM_GUI/model/types.py +24 -0
  62. AutoGLM_GUI/models/__init__.py +10 -0
  63. AutoGLM_GUI/models/history.py +96 -0
  64. AutoGLM_GUI/models/scheduled_task.py +71 -0
  65. AutoGLM_GUI/parsers/__init__.py +22 -0
  66. AutoGLM_GUI/parsers/base.py +50 -0
  67. AutoGLM_GUI/parsers/phone_parser.py +58 -0
  68. AutoGLM_GUI/phone_agent_manager.py +118 -367
  69. AutoGLM_GUI/platform_utils.py +31 -2
  70. AutoGLM_GUI/prompt_config.py +15 -0
  71. AutoGLM_GUI/prompts/__init__.py +32 -0
  72. AutoGLM_GUI/scheduler_manager.py +304 -0
  73. AutoGLM_GUI/schemas.py +272 -63
  74. AutoGLM_GUI/scrcpy_stream.py +159 -37
  75. AutoGLM_GUI/server.py +3 -1
  76. AutoGLM_GUI/socketio_server.py +114 -29
  77. AutoGLM_GUI/state.py +10 -30
  78. AutoGLM_GUI/static/assets/{about-DeclntHg.js → about-BQm96DAl.js} +1 -1
  79. AutoGLM_GUI/static/assets/alert-dialog-B42XxGPR.js +1 -0
  80. AutoGLM_GUI/static/assets/chat-C0L2gQYG.js +129 -0
  81. AutoGLM_GUI/static/assets/circle-alert-D4rSJh37.js +1 -0
  82. AutoGLM_GUI/static/assets/dialog-DZ78cEcj.js +45 -0
  83. AutoGLM_GUI/static/assets/history-DFBv7TGc.js +1 -0
  84. AutoGLM_GUI/static/assets/index-Bzyv2yQ2.css +1 -0
  85. AutoGLM_GUI/static/assets/{index-zQ4KKDHt.js → index-CmZSnDqc.js} +1 -1
  86. AutoGLM_GUI/static/assets/index-CssG-3TH.js +11 -0
  87. AutoGLM_GUI/static/assets/label-BCUzE_nm.js +1 -0
  88. AutoGLM_GUI/static/assets/logs-eoFxn5of.js +1 -0
  89. AutoGLM_GUI/static/assets/popover-DLsuV5Sx.js +1 -0
  90. AutoGLM_GUI/static/assets/scheduled-tasks-MyqGJvy_.js +1 -0
  91. AutoGLM_GUI/static/assets/square-pen-zGWYrdfj.js +1 -0
  92. AutoGLM_GUI/static/assets/textarea-BX6y7uM5.js +1 -0
  93. AutoGLM_GUI/static/assets/workflows-CYFs6ssC.js +1 -0
  94. AutoGLM_GUI/static/index.html +2 -2
  95. AutoGLM_GUI/types.py +142 -0
  96. {autoglm_gui-1.4.0.dist-info → autoglm_gui-1.5.0.dist-info}/METADATA +178 -92
  97. autoglm_gui-1.5.0.dist-info/RECORD +157 -0
  98. mai_agent/base.py +137 -0
  99. mai_agent/mai_grounding_agent.py +263 -0
  100. mai_agent/mai_naivigation_agent.py +526 -0
  101. mai_agent/prompt.py +148 -0
  102. mai_agent/unified_memory.py +67 -0
  103. mai_agent/utils.py +73 -0
  104. AutoGLM_GUI/api/dual_model.py +0 -311
  105. AutoGLM_GUI/dual_model/__init__.py +0 -53
  106. AutoGLM_GUI/dual_model/decision_model.py +0 -664
  107. AutoGLM_GUI/dual_model/dual_agent.py +0 -917
  108. AutoGLM_GUI/dual_model/protocols.py +0 -354
  109. AutoGLM_GUI/dual_model/vision_model.py +0 -442
  110. AutoGLM_GUI/mai_ui_adapter/agent_wrapper.py +0 -291
  111. AutoGLM_GUI/phone_agent_patches.py +0 -146
  112. AutoGLM_GUI/static/assets/chat-Iut2yhSw.js +0 -125
  113. AutoGLM_GUI/static/assets/dialog-BfdcBs1x.js +0 -45
  114. AutoGLM_GUI/static/assets/index-5hCCwHA7.css +0 -1
  115. AutoGLM_GUI/static/assets/index-DHF1NZh0.js +0 -12
  116. AutoGLM_GUI/static/assets/workflows-xiplap-r.js +0 -1
  117. autoglm_gui-1.4.0.dist-info/RECORD +0 -100
  118. {autoglm_gui-1.4.0.dist-info → autoglm_gui-1.5.0.dist-info}/WHEEL +0 -0
  119. {autoglm_gui-1.4.0.dist-info → autoglm_gui-1.5.0.dist-info}/entry_points.txt +0 -0
  120. {autoglm_gui-1.4.0.dist-info → autoglm_gui-1.5.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,292 @@
1
+ import json
2
+ import traceback
3
+ from typing import Any, Callable, cast
4
+
5
+ from openai import OpenAI
6
+
7
+ from AutoGLM_GUI.actions import ActionHandler, ActionResult
8
+ from AutoGLM_GUI.config import AgentConfig, ModelConfig, StepResult
9
+ from AutoGLM_GUI.device_protocol import DeviceProtocol
10
+ from AutoGLM_GUI.logger import logger
11
+ from AutoGLM_GUI.prompt_config import get_messages, get_system_prompt
12
+
13
+ from .message_builder import MessageBuilder
14
+ from .parser import GLMParser
15
+
16
+
17
+ class GLMAgent:
18
+ def __init__(
19
+ self,
20
+ model_config: ModelConfig,
21
+ agent_config: AgentConfig,
22
+ device: DeviceProtocol,
23
+ confirmation_callback: Callable[[str], bool] | None = None,
24
+ takeover_callback: Callable[[str], None] | None = None,
25
+ thinking_callback: Callable[[str], None] | None = None,
26
+ ):
27
+ self.model_config = model_config
28
+ self.agent_config = agent_config
29
+
30
+ self.openai_client = OpenAI(
31
+ base_url=model_config.base_url,
32
+ api_key=model_config.api_key,
33
+ timeout=120,
34
+ )
35
+ self.parser = GLMParser()
36
+
37
+ self.device = device
38
+ self.action_handler = ActionHandler(
39
+ device=self.device,
40
+ confirmation_callback=confirmation_callback,
41
+ takeover_callback=takeover_callback,
42
+ )
43
+
44
+ self._context: list[dict[str, Any]] = []
45
+ self._step_count = 0
46
+ self._is_running = False
47
+ self._thinking_callback = thinking_callback
48
+
49
+ def run(self, task: str) -> str:
50
+ self._context = []
51
+ self._step_count = 0
52
+ self._is_running = True
53
+
54
+ try:
55
+ result = self._execute_step(task, is_first=True)
56
+
57
+ if result.finished:
58
+ return result.message or "Task completed"
59
+
60
+ while self._step_count < self.agent_config.max_steps and self._is_running:
61
+ result = self._execute_step(is_first=False)
62
+
63
+ if result.finished:
64
+ return result.message or "Task completed"
65
+
66
+ return "Max steps reached"
67
+ finally:
68
+ self._is_running = False
69
+
70
+ def step(self, task: str | None = None) -> StepResult:
71
+ is_first = len(self._context) == 0
72
+
73
+ if is_first and not task:
74
+ raise ValueError("Task is required for the first step")
75
+
76
+ return self._execute_step(task, is_first)
77
+
78
+ def reset(self) -> None:
79
+ self._context = []
80
+ self._step_count = 0
81
+ self._is_running = False
82
+
83
+ def abort(self) -> None:
84
+ self._is_running = False
85
+ logger.info("Agent aborted by user")
86
+
87
+ def _stream_request(
88
+ self,
89
+ messages: list[dict[str, Any]],
90
+ on_thinking_chunk: Callable[[str], None] | None = None,
91
+ ) -> tuple[str, str, str]:
92
+ stream = self.openai_client.chat.completions.create(
93
+ messages=cast(Any, messages),
94
+ model=self.model_config.model_name,
95
+ max_tokens=self.model_config.max_tokens,
96
+ temperature=self.model_config.temperature,
97
+ top_p=self.model_config.top_p,
98
+ frequency_penalty=self.model_config.frequency_penalty,
99
+ extra_body=self.model_config.extra_body,
100
+ stream=True,
101
+ )
102
+
103
+ raw_content = ""
104
+ buffer = ""
105
+ action_markers = ["finish(message=", "do(action="]
106
+ in_action_phase = False
107
+
108
+ for chunk in stream:
109
+ if len(chunk.choices) == 0:
110
+ continue
111
+ if chunk.choices[0].delta.content is not None:
112
+ content = chunk.choices[0].delta.content
113
+ raw_content += content
114
+
115
+ if in_action_phase:
116
+ continue
117
+
118
+ buffer += content
119
+
120
+ marker_found = False
121
+ for marker in action_markers:
122
+ if marker in buffer:
123
+ thinking_part = buffer.split(marker, 1)[0]
124
+ if on_thinking_chunk:
125
+ on_thinking_chunk(thinking_part)
126
+ in_action_phase = True
127
+ marker_found = True
128
+ break
129
+
130
+ if marker_found:
131
+ continue
132
+
133
+ is_potential_marker = False
134
+ for marker in action_markers:
135
+ for i in range(1, len(marker)):
136
+ if buffer.endswith(marker[:i]):
137
+ is_potential_marker = True
138
+ break
139
+ if is_potential_marker:
140
+ break
141
+
142
+ if not is_potential_marker:
143
+ if on_thinking_chunk:
144
+ on_thinking_chunk(buffer)
145
+ buffer = ""
146
+
147
+ thinking, action = self._parse_raw_response(raw_content)
148
+ return thinking, action, raw_content
149
+
150
+ def _parse_raw_response(self, content: str) -> tuple[str, str]:
151
+ if "finish(message=" in content:
152
+ parts = content.split("finish(message=", 1)
153
+ thinking = parts[0].strip()
154
+ action = "finish(message=" + parts[1]
155
+ return thinking, action
156
+
157
+ if "do(action=" in content:
158
+ parts = content.split("do(action=", 1)
159
+ thinking = parts[0].strip()
160
+ action = "do(action=" + parts[1]
161
+ return thinking, action
162
+
163
+ if "<answer>" in content:
164
+ parts = content.split("<answer>", 1)
165
+ thinking = parts[0].replace("<think>", "").replace("</think>", "").strip()
166
+ action = parts[1].replace("</answer>", "").strip()
167
+ return thinking, action
168
+
169
+ return "", content
170
+
171
+ def _execute_step(
172
+ self, user_prompt: str | None = None, is_first: bool = False
173
+ ) -> StepResult:
174
+ self._step_count += 1
175
+
176
+ screenshot = self.device.get_screenshot()
177
+ current_app = self.device.get_current_app()
178
+
179
+ if is_first:
180
+ system_prompt = self.agent_config.system_prompt
181
+ if system_prompt is None:
182
+ system_prompt = get_system_prompt(self.agent_config.lang)
183
+
184
+ self._context.append(MessageBuilder.create_system_message(system_prompt))
185
+
186
+ screen_info = MessageBuilder.build_screen_info(current_app)
187
+ text_content = f"{user_prompt}\n\n{screen_info}"
188
+
189
+ self._context.append(
190
+ MessageBuilder.create_user_message(
191
+ text=text_content, image_base64=screenshot.base64_data
192
+ )
193
+ )
194
+ else:
195
+ screen_info = MessageBuilder.build_screen_info(current_app)
196
+ text_content = f"** Screen Info **\n\n{screen_info}"
197
+
198
+ self._context.append(
199
+ MessageBuilder.create_user_message(
200
+ text=text_content, image_base64=screenshot.base64_data
201
+ )
202
+ )
203
+
204
+ try:
205
+ msgs = get_messages(self.agent_config.lang)
206
+ if self.agent_config.verbose:
207
+ print("\n" + "=" * 50)
208
+ print(f"💭 {msgs['thinking']}:")
209
+ print("-" * 50)
210
+
211
+ callback = self._thinking_callback
212
+ if callback is None and self.agent_config.verbose:
213
+
214
+ def print_chunk(chunk: str) -> None:
215
+ print(chunk, end="", flush=True)
216
+
217
+ callback = print_chunk
218
+
219
+ thinking, action_str, raw_content = self._stream_request(
220
+ self._context, on_thinking_chunk=callback
221
+ )
222
+ except Exception as e:
223
+ if self.agent_config.verbose:
224
+ traceback.print_exc()
225
+ return StepResult(
226
+ success=False,
227
+ finished=True,
228
+ action=None,
229
+ thinking="",
230
+ message=f"Model error: {e}",
231
+ )
232
+
233
+ try:
234
+ action = self.parser.parse(action_str)
235
+ except ValueError as e:
236
+ if self.agent_config.verbose:
237
+ logger.warning(f"Failed to parse action: {e}, treating as finish")
238
+ action = {"_metadata": "finish", "message": action_str}
239
+
240
+ if self.agent_config.verbose:
241
+ print()
242
+ print("-" * 50)
243
+ print(f"🎯 {msgs['action']}:")
244
+ print(json.dumps(action, ensure_ascii=False, indent=2))
245
+ print("=" * 50 + "\n")
246
+
247
+ self._context[-1] = MessageBuilder.remove_images_from_message(self._context[-1])
248
+
249
+ try:
250
+ result = self.action_handler.execute(
251
+ action, screenshot.width, screenshot.height
252
+ )
253
+ except Exception as e:
254
+ if self.agent_config.verbose:
255
+ traceback.print_exc()
256
+ result = ActionResult(success=False, should_finish=True, message=str(e))
257
+
258
+ self._context.append(
259
+ MessageBuilder.create_assistant_message(
260
+ f"<think>{thinking}</think><answer>{action_str}</answer>"
261
+ )
262
+ )
263
+
264
+ finished = action.get("_metadata") == "finish" or result.should_finish
265
+
266
+ if finished and self.agent_config.verbose:
267
+ msgs = get_messages(self.agent_config.lang)
268
+ print("\n" + "🎉 " + "=" * 48)
269
+ print(
270
+ f"✅ {msgs['task_completed']}: {result.message or action.get('message', msgs['done'])}"
271
+ )
272
+ print("=" * 50 + "\n")
273
+
274
+ return StepResult(
275
+ success=result.success,
276
+ finished=finished,
277
+ action=action,
278
+ thinking=thinking,
279
+ message=result.message or action.get("message"),
280
+ )
281
+
282
+ @property
283
+ def context(self) -> list[dict[str, Any]]:
284
+ return self._context.copy()
285
+
286
+ @property
287
+ def step_count(self) -> int:
288
+ return self._step_count
289
+
290
+ @property
291
+ def is_running(self) -> bool:
292
+ return self._is_running
@@ -0,0 +1,81 @@
1
+ """Message builder for GLM agent - copied from phone_agent.model.client.
2
+
3
+ This is an exact copy of the upstream MessageBuilder to ensure consistent behavior.
4
+ """
5
+
6
+ import json
7
+ from typing import Any
8
+
9
+
10
+ class MessageBuilder:
11
+ """Helper class for building conversation messages."""
12
+
13
+ @staticmethod
14
+ def create_system_message(content: str) -> dict[str, Any]:
15
+ """Create a system message."""
16
+ return {"role": "system", "content": content}
17
+
18
+ @staticmethod
19
+ def create_user_message(
20
+ text: str, image_base64: str | None = None
21
+ ) -> dict[str, Any]:
22
+ """
23
+ Create a user message with optional image.
24
+
25
+ Args:
26
+ text: Text content.
27
+ image_base64: Optional base64-encoded image.
28
+
29
+ Returns:
30
+ Message dictionary.
31
+ """
32
+ content = []
33
+
34
+ if image_base64:
35
+ content.append(
36
+ {
37
+ "type": "image_url",
38
+ "image_url": {"url": f"data:image/png;base64,{image_base64}"},
39
+ }
40
+ )
41
+
42
+ content.append({"type": "text", "text": text})
43
+
44
+ return {"role": "user", "content": content}
45
+
46
+ @staticmethod
47
+ def create_assistant_message(content: str) -> dict[str, Any]:
48
+ """Create an assistant message."""
49
+ return {"role": "assistant", "content": content}
50
+
51
+ @staticmethod
52
+ def remove_images_from_message(message: dict[str, Any]) -> dict[str, Any]:
53
+ """
54
+ Remove image content from a message to save context space.
55
+
56
+ Args:
57
+ message: Message dictionary.
58
+
59
+ Returns:
60
+ Message with images removed.
61
+ """
62
+ if isinstance(message.get("content"), list):
63
+ message["content"] = [
64
+ item for item in message["content"] if item.get("type") == "text"
65
+ ]
66
+ return message
67
+
68
+ @staticmethod
69
+ def build_screen_info(current_app: str, **extra_info) -> str:
70
+ """
71
+ Build screen info string for the model.
72
+
73
+ Args:
74
+ current_app: Current app name.
75
+ **extra_info: Additional info to include.
76
+
77
+ Returns:
78
+ JSON string with screen info.
79
+ """
80
+ info = {"current_app": current_app, **extra_info}
81
+ return json.dumps(info, ensure_ascii=False)
@@ -0,0 +1,110 @@
1
+ import ast
2
+ from typing import Any
3
+
4
+
5
+ class GLMParser:
6
+ @property
7
+ def coordinate_scale(self) -> int:
8
+ return 1000
9
+
10
+ def parse(self, raw_response: str) -> dict[str, Any]:
11
+ action_str = raw_response.strip()
12
+
13
+ if action_str.startswith("finish("):
14
+ return self._parse_finish(action_str)
15
+ if action_str.startswith("do("):
16
+ return self._parse_do(action_str)
17
+ raise ValueError(f"Unknown action format: {action_str}")
18
+
19
+ def _parse_finish(self, action_str: str) -> dict[str, Any]:
20
+ try:
21
+ params = self._extract_params(action_str, "finish")
22
+ return {
23
+ "_metadata": "finish",
24
+ "message": params.get("message", "Task completed"),
25
+ }
26
+ except Exception as e:
27
+ raise ValueError(f"Failed to parse finish action: {e}") from e
28
+
29
+ def _parse_do(self, action_str: str) -> dict[str, Any]:
30
+ try:
31
+ params = self._extract_params(action_str, "do")
32
+ action_name = params.get("action", "")
33
+
34
+ result = {
35
+ "_metadata": "do",
36
+ "action": action_name,
37
+ }
38
+
39
+ for key, value in params.items():
40
+ if key != "action":
41
+ result[key] = value
42
+
43
+ return result
44
+ except Exception as e:
45
+ raise ValueError(f"Failed to parse do action: {e}") from e
46
+
47
+ def _extract_params(self, action_str: str, function_name: str) -> dict[str, Any]:
48
+ prefix = f"{function_name}("
49
+ if not action_str.startswith(prefix):
50
+ raise ValueError(f"Action does not start with {prefix}")
51
+
52
+ params_str = action_str[len(prefix) : -1]
53
+
54
+ params: dict[str, Any] = {}
55
+ current_key = None
56
+ current_value = ""
57
+ in_quotes = False
58
+ quote_char = None
59
+ bracket_depth = 0
60
+ i = 0
61
+
62
+ while i < len(params_str):
63
+ char = params_str[i]
64
+
65
+ if char in ('"', "'") and (i == 0 or params_str[i - 1] != "\\"):
66
+ if not in_quotes:
67
+ in_quotes = True
68
+ quote_char = char
69
+ elif char == quote_char:
70
+ in_quotes = False
71
+ quote_char = None
72
+
73
+ if not in_quotes:
74
+ if char in ("[", "{"):
75
+ bracket_depth += 1
76
+ elif char in ("]", "}"):
77
+ bracket_depth -= 1
78
+
79
+ if char == "=" and bracket_depth == 0:
80
+ current_key = current_value.strip()
81
+ current_value = ""
82
+ i += 1
83
+ continue
84
+
85
+ if char == "," and bracket_depth == 0:
86
+ if current_key:
87
+ params[current_key] = self._parse_value(current_value.strip())
88
+ current_key = None
89
+ current_value = ""
90
+ i += 1
91
+ continue
92
+
93
+ current_value += char
94
+ i += 1
95
+
96
+ if current_key:
97
+ params[current_key] = self._parse_value(current_value.strip())
98
+
99
+ return params
100
+
101
+ def _parse_value(self, value_str: str) -> Any:
102
+ value_str = value_str.strip()
103
+
104
+ if not value_str:
105
+ return ""
106
+
107
+ try:
108
+ return ast.literal_eval(value_str)
109
+ except (ValueError, SyntaxError):
110
+ return value_str
@@ -0,0 +1,77 @@
1
+ from datetime import datetime
2
+
3
+ today = datetime.today()
4
+ formatted_date = today.strftime("%Y-%m-%d, %A")
5
+
6
+ SYSTEM_PROMPT = (
7
+ "The current date: "
8
+ + formatted_date
9
+ + """
10
+ # Setup
11
+ You are a professional Android operation agent assistant that can fulfill the user's high-level instructions. Given a screenshot of the Android interface at each step, you first analyze the situation, then plan the best course of action using Python-style pseudo-code.
12
+
13
+ # More details about the code
14
+ Your response format must be structured as follows:
15
+
16
+ Think first: Use <think>...</think> to analyze the current screen, identify key elements, and determine the most efficient action.
17
+ Provide the action: Use <answer>...</answer> to return a single line of pseudo-code representing the operation.
18
+
19
+ Your output should STRICTLY follow the format:
20
+ <think>
21
+ [Your thought]
22
+ </think>
23
+ <answer>
24
+ [Your operation code]
25
+ </answer>
26
+
27
+ - **Tap**
28
+ Perform a tap action on a specified screen area. The element is a list of 2 integers, representing the coordinates of the tap point.
29
+ **Example**:
30
+ <answer>
31
+ do(action=\"Tap\", element=[x,y])
32
+ </answer>
33
+ - **Type**
34
+ Enter text into the currently focused input field.
35
+ **Example**:
36
+ <answer>
37
+ do(action=\"Type\", text=\"Hello World\")
38
+ </answer>
39
+ - **Swipe**
40
+ Perform a swipe action with start point and end point.
41
+ **Examples**:
42
+ <answer>
43
+ do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])
44
+ </answer>
45
+ - **Long Press**
46
+ Perform a long press action on a specified screen area.
47
+ You can add the element to the action to specify the long press area. The element is a list of 2 integers, representing the coordinates of the long press point.
48
+ **Example**:
49
+ <answer>
50
+ do(action=\"Long Press\", element=[x,y])
51
+ </answer>
52
+ - **Launch**
53
+ Launch an app. Try to use launch action when you need to launch an app. Check the instruction to choose the right app before you use this action.
54
+ **Example**:
55
+ <answer>
56
+ do(action=\"Launch\", app=\"Settings\")
57
+ </answer>
58
+ - **Back**
59
+ Press the Back button to navigate to the previous screen.
60
+ **Example**:
61
+ <answer>
62
+ do(action=\"Back\")
63
+ </answer>
64
+ - **Finish**
65
+ Terminate the program and optionally print a message.
66
+ **Example**:
67
+ <answer>
68
+ finish(message=\"Task completed.\")
69
+ </answer>
70
+
71
+
72
+ REMEMBER:
73
+ - Think before you act: Always analyze the current UI and the best course of action before executing any step, and output in <think> part.
74
+ - Only ONE LINE of action in <answer> part per response: Each step must contain exactly one line of executable code.
75
+ - Generate execution code strictly according to format requirements.
76
+ """
77
+ )
@@ -0,0 +1,75 @@
1
+ from datetime import datetime
2
+
3
+ today = datetime.today()
4
+ weekday_names = ["星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"]
5
+ weekday = weekday_names[today.weekday()]
6
+ formatted_date = f"{today.year}年{today.month:02d}月{today.day:02d}日 {weekday}"
7
+
8
+ SYSTEM_PROMPT = (
9
+ "今天的日期是: "
10
+ + formatted_date
11
+ + """
12
+ 你是一个智能体分析专家,可以根据操作历史和当前状态图执行一系列操作来完成任务。
13
+ 你必须严格按照要求输出以下格式:
14
+ <think>{think}</think>
15
+ <answer>{action}</answer>
16
+
17
+ 其中:
18
+ - {think} 是对你为什么选择这个操作的简短推理说明。
19
+ - {action} 是本次执行的具体操作指令,必须严格遵循下方定义的指令格式。
20
+
21
+ 操作指令及其作用如下:
22
+ - do(action=\"Launch\", app=\"xxx\")
23
+ Launch是启动目标app的操作,这比通过主屏幕导航更快。此操作完成后,您将自动收到结果状态的截图。
24
+ - do(action=\"Tap\", element=[x,y])
25
+ Tap是点击操作,点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序,或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角(999,999)结束。此操作完成后,您将自动收到结果状态的截图。
26
+ - do(action=\"Tap\", element=[x,y], message=\"重要操作\")
27
+ 基本功能同Tap,点击涉及财产、支付、隐私等敏感按钮时触发。
28
+ - do(action=\"Type\", text=\"xxx\")
29
+ Type是输入操作,在当前聚焦的输入框中输入文本。使用此操作前,请确保输入框已被聚焦(先点击它)。输入的文本将像使用键盘输入一样输入。重要提示:手机可能正在使用 ADB 键盘,该键盘不会像普通键盘那样占用屏幕空间。要确认键盘已激活,请查看屏幕底部是否显示 'ADB Keyboard {ON}' 类似的文本,或者检查输入框是否处于激活/高亮状态。不要仅仅依赖视觉上的键盘显示。自动清除文本:当你使用输入操作时,输入框中现有的任何文本(包括占位符文本和实际输入)都会在输入新文本前自动清除。你无需在输入前手动清除文本——直接使用输入操作输入所需文本即可。操作完成后,你将自动收到结果状态的截图。
30
+ - do(action=\"Type_Name\", text=\"xxx\")
31
+ Type_Name是输入人名的操作,基本功能同Type。
32
+ - do(action=\"Interact\")
33
+ Interact是当有多个满足条件的选项时而触发的交互操作,询问用户如何选择。
34
+ - do(action=\"Swipe\", start=[x1,y1], end=[x2,y2])
35
+ Swipe是滑动操作,通过从起始坐标拖动到结束坐标来执行滑动手势。可用于滚动内容、在屏幕之间导航、下拉通知栏以及项目栏或进行基于手势的导航。坐标系统从左上角 (0,0) 开始到右下角(999,999)结束。滑动持续时间会自动调整以实现自然的移动。此操作完成后,您将自动收到结果状态的截图。
36
+ - do(action=\"Note\", message=\"True\")
37
+ 记录当前页面内容以便后续总结。
38
+ - do(action=\"Call_API\", instruction=\"xxx\")
39
+ 总结或评论当前页面或已记录的内容。
40
+ - do(action=\"Long Press\", element=[x,y])
41
+ Long Pres是长按操作,在屏幕上的特定点长按指定时间。可用于触发上下文菜单、选择文本或激活长按交互。坐标系统从左上角 (0,0) 开始到右下角(999,999)结束。此操作完成后,您将自动收到结果状态的屏幕截图。
42
+ - do(action=\"Double Tap\", element=[x,y])
43
+ Double Tap在屏幕上的特定点快速连续点按两次。使用此操作可以激活双击交互,如缩放、选择文本或打开项目。坐标系统从左上角 (0,0) 开始到右下角(999,999)结束。此操作完成后,您将自动收到结果状态的截图。
44
+ - do(action=\"Take_over\", message=\"xxx\")
45
+ Take_over是接管操作,表示在登录和验证阶段需要用户协助。
46
+ - do(action=\"Back\")
47
+ 导航返回到上一个屏幕或关闭当前对话框。相当于按下 Android 的返回按钮。使用此操作可以从更深的屏幕返回、关闭弹出窗口或退出当前上下文。此操作完成后,您将自动收到结果状态的截图。
48
+ - do(action=\"Home\")
49
+ Home是回到系统桌面的操作,相当于按下 Android 主屏幕按钮。使用此操作可退出当前应用并返回启动器,或从已知状态启动新任务。此操作完成后,您将自动收到结果状态的截图。
50
+ - do(action=\"Wait\", duration=\"x seconds\")
51
+ 等待页面加载,x为需要等待多少秒。
52
+ - finish(message=\"xxx\")
53
+ finish是结束任务的操作,表示准确完整完成任务,message是终止信息。
54
+
55
+ 必须遵循的规则:
56
+ 1. 在执行任何操作前,先检查当前app是否是目标app,如果不是,先执行 Launch。
57
+ 2. 如果进入到了无关页面,先执行 Back。如果执行Back后页面没有变化,请点击页面左上角的返回键进行返回,或者右上角的X号关闭。
58
+ 3. 如果页面未加载出内容,最多连续 Wait 三次,否则执行 Back重新进入。
59
+ 4. 如果页面显示网络问题,需要重新加载,请点击重新加载。
60
+ 5. 如果当前页面找不到目标联系人、商品、店铺等信息,可以尝试 Swipe 滑动查找。
61
+ 6. 遇到价格区间、时间区间等筛选条件,如果没有完全符合的,可以放宽要求。
62
+ 7. 在做小红书总结类任务时一定要筛选图文笔记。
63
+ 8. 购物车全选后再点击全选可以把状态设为全不选,在做购物车任务时,如果购物车里已经有商品被选中时,你需要点击全选后再点击取消全选,再去找需要购买或者删除的商品。
64
+ 9. 在做外卖任务时,如果相应店铺购物车里已经有其他商品你需要先把购物车清空再去购买用户指定的外卖。
65
+ 10. 在做点外卖任务时,如果用户需要点多个外卖,请尽量在同一店铺进行购买,如果无法找到可以下单,并说明某个商品未找到。
66
+ 11. 请严格遵循用户意图执行任务,用户的特殊要求可以执行多次搜索,滑动查找。比如(i)用户要求点一杯咖啡,要咸的,你可以直接搜索咸咖啡,或者搜索咖啡后滑动查找咸的咖啡,比如海盐咖啡。(ii)用户要找到XX群,发一条消息,你可以先搜索XX群,找不到结果后,将\"群\"字去掉,搜索XX重试。(iii)用户要找到宠物友好的餐厅,你可以搜索餐厅,找到筛选,找到设施,选择可带宠物,或者直接搜索可带宠物,必要时可以使用AI搜索。
67
+ 12. 在选择日期时,如果原滑动方向与预期日期越来越远,请向反方向滑动查找。
68
+ 13. 执行任务过程中如果有多个可选择的项目栏,请逐个查找每个项目栏,直到完成任务,一定不要在同一项目栏多次查找,从而陷入死循环。
69
+ 14. 在执行下一步操作前请一定要检查上一步的操作是否生效,如果点击没生效,可能因为app反应较慢,请先稍微等待一下,如果还是不生效请调整一下点击位置重试,如果仍然不生效请跳过这一步继续任务,并在finish message说明点击不生效。
70
+ 15. 在执行任务中如果遇到滑动不生效的情况,请调整一下起始点位置,增大滑动距离重试,如果还是不生效,有可能是已经滑到底了,请继续向反方向滑动,直到顶部或底部,如果仍然没有符合要求的结果,请跳过这一步继续任务,并在finish message说明但没找到要求的项目。
71
+ 16. 在做游戏任务时如果在战斗页面如果有自动战斗一定要开启自动战斗,如果多轮历史状态相似要检查自动战斗是否开启。
72
+ 17. 如果没有合适的搜索结果,可能是因为搜索页面不对,请返回到搜索页面的上一级尝试重新搜索,如果尝试三次返回上一级搜索后仍然没有符合要求的结果,执行 finish(message=\"原因\")。
73
+ 18. 在结束任务前请一定要仔细检查任务是否完整准确的完成,如果出现错选、漏选、多选的情况,请返回之前的步骤进行纠正。
74
+ """
75
+ )
@@ -0,0 +1,28 @@
1
+ """MAI Agent - Internal implementation.
2
+
3
+ This module co-locates all MAI-specific code:
4
+ - InternalMAIAgent: The main agent implementation
5
+ - MAIParser: XML+JSON format parser with coordinate conversion (0-999 to 0-1000)
6
+ - MAI_MOBILE_SYSTEM_PROMPT: System prompt for Chinese environments
7
+ - TrajMemory, TrajStep: Trajectory memory for multi-step tasks
8
+
9
+ Design notes:
10
+ - MAI uses 0-999 coordinate system (normalized internally)
11
+ - Supports multi-image history context (configurable via history_n)
12
+ - Chinese-optimized prompts for domestic app scenarios
13
+ - Internal implementation replacing third-party mai_agent dependency
14
+ """
15
+
16
+ from .agent import InternalMAIAgent
17
+ from .parser import MAIParser, MAIParseError
18
+ from .prompts import MAI_MOBILE_SYSTEM_PROMPT
19
+ from .traj_memory import TrajMemory, TrajStep
20
+
21
+ __all__ = [
22
+ "InternalMAIAgent",
23
+ "MAIParser",
24
+ "MAIParseError",
25
+ "MAI_MOBILE_SYSTEM_PROMPT",
26
+ "TrajMemory",
27
+ "TrajStep",
28
+ ]