mobile-mcp-ai 2.5.8__py3-none-any.whl → 2.6.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/config.py +32 -0
- mobile_mcp/core/basic_tools_lite.py +1669 -667
- mobile_mcp/core/mobile_client.py +23 -2
- mobile_mcp/mcp_tools/mcp_server.py +298 -278
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/METADATA +1 -1
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/RECORD +10 -10
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/WHEEL +0 -0
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/entry_points.txt +0 -0
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/licenses/LICENSE +0 -0
- {mobile_mcp_ai-2.5.8.dist-info → mobile_mcp_ai-2.6.5.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
- 核心功能精简
|
|
9
9
|
- 保留 pytest 脚本生成
|
|
10
10
|
- 支持操作历史记录
|
|
11
|
+
- Token 优化模式(省钱)
|
|
11
12
|
"""
|
|
12
13
|
|
|
13
14
|
import asyncio
|
|
@@ -17,6 +18,19 @@ from pathlib import Path
|
|
|
17
18
|
from typing import Dict, List, Optional
|
|
18
19
|
from datetime import datetime
|
|
19
20
|
|
|
21
|
+
# Token 优化配置(只精简格式,不限制数量,确保准确度)
|
|
22
|
+
try:
|
|
23
|
+
from mobile_mcp.config import Config
|
|
24
|
+
TOKEN_OPTIMIZATION = Config.TOKEN_OPTIMIZATION_ENABLED
|
|
25
|
+
MAX_ELEMENTS = Config.MAX_ELEMENTS_RETURN
|
|
26
|
+
MAX_SOM_ELEMENTS = Config.MAX_SOM_ELEMENTS_RETURN
|
|
27
|
+
COMPACT_RESPONSE = Config.COMPACT_RESPONSE
|
|
28
|
+
except ImportError:
|
|
29
|
+
TOKEN_OPTIMIZATION = True
|
|
30
|
+
MAX_ELEMENTS = 0 # 0 = 不限制
|
|
31
|
+
MAX_SOM_ELEMENTS = 0 # 0 = 不限制
|
|
32
|
+
COMPACT_RESPONSE = True
|
|
33
|
+
|
|
20
34
|
|
|
21
35
|
class BasicMobileToolsLite:
|
|
22
36
|
"""精简版移动端工具"""
|
|
@@ -31,6 +45,9 @@ class BasicMobileToolsLite:
|
|
|
31
45
|
|
|
32
46
|
# 操作历史(用于生成 pytest 脚本)
|
|
33
47
|
self.operation_history: List[Dict] = []
|
|
48
|
+
|
|
49
|
+
# 目标应用包名(用于监测应用跳转)
|
|
50
|
+
self.target_package: Optional[str] = None
|
|
34
51
|
|
|
35
52
|
def _is_ios(self) -> bool:
|
|
36
53
|
"""判断当前是否为 iOS 平台"""
|
|
@@ -45,7 +62,7 @@ class BasicMobileToolsLite:
|
|
|
45
62
|
return None
|
|
46
63
|
|
|
47
64
|
def _record_operation(self, action: str, **kwargs):
|
|
48
|
-
"""
|
|
65
|
+
"""记录操作到历史(旧接口,保持兼容)"""
|
|
49
66
|
record = {
|
|
50
67
|
'action': action,
|
|
51
68
|
'timestamp': datetime.now().isoformat(),
|
|
@@ -53,6 +70,231 @@ class BasicMobileToolsLite:
|
|
|
53
70
|
}
|
|
54
71
|
self.operation_history.append(record)
|
|
55
72
|
|
|
73
|
+
def _record_click(self, locator_type: str, locator_value: str,
|
|
74
|
+
x_percent: float = 0, y_percent: float = 0,
|
|
75
|
+
element_desc: str = '', locator_attr: str = ''):
|
|
76
|
+
"""记录点击操作(标准格式)
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
locator_type: 定位类型 'text' | 'id' | 'percent' | 'coords'
|
|
80
|
+
locator_value: 定位值(文本内容、resource-id、或坐标描述)
|
|
81
|
+
x_percent: 百分比 X 坐标(兜底方案)
|
|
82
|
+
y_percent: 百分比 Y 坐标(兜底方案)
|
|
83
|
+
element_desc: 元素描述(用于脚本注释)
|
|
84
|
+
locator_attr: Android 选择器属性 'text'|'textContains'|'description'|'descriptionContains'
|
|
85
|
+
"""
|
|
86
|
+
record = {
|
|
87
|
+
'action': 'click',
|
|
88
|
+
'timestamp': datetime.now().isoformat(),
|
|
89
|
+
'locator_type': locator_type,
|
|
90
|
+
'locator_value': locator_value,
|
|
91
|
+
'locator_attr': locator_attr or locator_type, # 默认与 type 相同
|
|
92
|
+
'x_percent': x_percent,
|
|
93
|
+
'y_percent': y_percent,
|
|
94
|
+
'element_desc': element_desc or locator_value,
|
|
95
|
+
}
|
|
96
|
+
self.operation_history.append(record)
|
|
97
|
+
|
|
98
|
+
def _record_long_press(self, locator_type: str, locator_value: str,
|
|
99
|
+
duration: float = 1.0,
|
|
100
|
+
x_percent: float = 0, y_percent: float = 0,
|
|
101
|
+
element_desc: str = '', locator_attr: str = ''):
|
|
102
|
+
"""记录长按操作(标准格式)"""
|
|
103
|
+
record = {
|
|
104
|
+
'action': 'long_press',
|
|
105
|
+
'timestamp': datetime.now().isoformat(),
|
|
106
|
+
'locator_type': locator_type,
|
|
107
|
+
'locator_value': locator_value,
|
|
108
|
+
'locator_attr': locator_attr or locator_type,
|
|
109
|
+
'duration': duration,
|
|
110
|
+
'x_percent': x_percent,
|
|
111
|
+
'y_percent': y_percent,
|
|
112
|
+
'element_desc': element_desc or locator_value,
|
|
113
|
+
}
|
|
114
|
+
self.operation_history.append(record)
|
|
115
|
+
|
|
116
|
+
def _record_input(self, text: str, locator_type: str = '', locator_value: str = '',
|
|
117
|
+
x_percent: float = 0, y_percent: float = 0):
|
|
118
|
+
"""记录输入操作(标准格式)"""
|
|
119
|
+
record = {
|
|
120
|
+
'action': 'input',
|
|
121
|
+
'timestamp': datetime.now().isoformat(),
|
|
122
|
+
'text': text,
|
|
123
|
+
'locator_type': locator_type,
|
|
124
|
+
'locator_value': locator_value,
|
|
125
|
+
'x_percent': x_percent,
|
|
126
|
+
'y_percent': y_percent,
|
|
127
|
+
}
|
|
128
|
+
self.operation_history.append(record)
|
|
129
|
+
|
|
130
|
+
def _record_swipe(self, direction: str):
|
|
131
|
+
"""记录滑动操作"""
|
|
132
|
+
record = {
|
|
133
|
+
'action': 'swipe',
|
|
134
|
+
'timestamp': datetime.now().isoformat(),
|
|
135
|
+
'direction': direction,
|
|
136
|
+
}
|
|
137
|
+
self.operation_history.append(record)
|
|
138
|
+
|
|
139
|
+
def _record_key(self, key: str):
|
|
140
|
+
"""记录按键操作"""
|
|
141
|
+
record = {
|
|
142
|
+
'action': 'press_key',
|
|
143
|
+
'timestamp': datetime.now().isoformat(),
|
|
144
|
+
'key': key,
|
|
145
|
+
}
|
|
146
|
+
self.operation_history.append(record)
|
|
147
|
+
|
|
148
|
+
def _get_current_package(self) -> Optional[str]:
|
|
149
|
+
"""获取当前前台应用的包名/Bundle ID"""
|
|
150
|
+
try:
|
|
151
|
+
if self._is_ios():
|
|
152
|
+
ios_client = self._get_ios_client()
|
|
153
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
154
|
+
app_info = ios_client.wda.session().app_current()
|
|
155
|
+
return app_info.get('bundleId')
|
|
156
|
+
else:
|
|
157
|
+
info = self.client.u2.app_current()
|
|
158
|
+
return info.get('package')
|
|
159
|
+
except Exception:
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
def _check_app_switched(self) -> Dict:
|
|
163
|
+
"""检查是否已跳出目标应用
|
|
164
|
+
|
|
165
|
+
Returns:
|
|
166
|
+
{
|
|
167
|
+
'switched': bool, # 是否跳转
|
|
168
|
+
'current_package': str, # 当前应用包名
|
|
169
|
+
'target_package': str, # 目标应用包名
|
|
170
|
+
'message': str # 提示信息
|
|
171
|
+
}
|
|
172
|
+
"""
|
|
173
|
+
if not self.target_package:
|
|
174
|
+
return {
|
|
175
|
+
'switched': False,
|
|
176
|
+
'current_package': None,
|
|
177
|
+
'target_package': None,
|
|
178
|
+
'message': '⚠️ 未设置目标应用,无法监测应用跳转'
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
current = self._get_current_package()
|
|
182
|
+
if not current:
|
|
183
|
+
return {
|
|
184
|
+
'switched': False,
|
|
185
|
+
'current_package': None,
|
|
186
|
+
'target_package': self.target_package,
|
|
187
|
+
'message': '⚠️ 无法获取当前应用包名'
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if current != self.target_package:
|
|
191
|
+
return {
|
|
192
|
+
'switched': True,
|
|
193
|
+
'current_package': current,
|
|
194
|
+
'target_package': self.target_package,
|
|
195
|
+
'message': f'⚠️ 应用已跳转!当前应用: {current},目标应用: {self.target_package}'
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
return {
|
|
199
|
+
'switched': False,
|
|
200
|
+
'current_package': current,
|
|
201
|
+
'target_package': self.target_package,
|
|
202
|
+
'message': f'✅ 仍在目标应用: {current}'
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
def _return_to_target_app(self) -> Dict:
|
|
206
|
+
"""返回到目标应用
|
|
207
|
+
|
|
208
|
+
策略:
|
|
209
|
+
1. 先按返回键(可能关闭弹窗或返回上一页)
|
|
210
|
+
2. 如果还在其他应用,启动目标应用
|
|
211
|
+
3. 验证是否成功返回
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
{
|
|
215
|
+
'success': bool,
|
|
216
|
+
'message': str,
|
|
217
|
+
'method': str # 使用的返回方法
|
|
218
|
+
}
|
|
219
|
+
"""
|
|
220
|
+
if not self.target_package:
|
|
221
|
+
return {
|
|
222
|
+
'success': False,
|
|
223
|
+
'message': '❌ 未设置目标应用,无法返回',
|
|
224
|
+
'method': None
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
# 先检查当前应用
|
|
229
|
+
current = self._get_current_package()
|
|
230
|
+
if not current:
|
|
231
|
+
return {
|
|
232
|
+
'success': False,
|
|
233
|
+
'message': '❌ 无法获取当前应用包名',
|
|
234
|
+
'method': None
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
# 如果已经在目标应用,不需要返回
|
|
238
|
+
if current == self.target_package:
|
|
239
|
+
return {
|
|
240
|
+
'success': True,
|
|
241
|
+
'message': f'✅ 已在目标应用: {self.target_package}',
|
|
242
|
+
'method': 'already_in_target'
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
# 策略1: 先按返回键(可能关闭弹窗或返回)
|
|
246
|
+
if self._is_ios():
|
|
247
|
+
ios_client = self._get_ios_client()
|
|
248
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
249
|
+
# iOS 返回键
|
|
250
|
+
ios_client.wda.press('home') # iOS 先按 home
|
|
251
|
+
time.sleep(0.5)
|
|
252
|
+
# 然后启动目标应用
|
|
253
|
+
ios_client.wda.app_activate(self.target_package)
|
|
254
|
+
else:
|
|
255
|
+
return {
|
|
256
|
+
'success': False,
|
|
257
|
+
'message': '❌ iOS 客户端未初始化',
|
|
258
|
+
'method': None
|
|
259
|
+
}
|
|
260
|
+
else:
|
|
261
|
+
# Android: 先按返回键
|
|
262
|
+
self.client.u2.press('back')
|
|
263
|
+
time.sleep(0.5)
|
|
264
|
+
|
|
265
|
+
# 检查是否已返回
|
|
266
|
+
current = self._get_current_package()
|
|
267
|
+
if current == self.target_package:
|
|
268
|
+
return {
|
|
269
|
+
'success': True,
|
|
270
|
+
'message': f'✅ 已返回目标应用: {self.target_package}(通过返回键)',
|
|
271
|
+
'method': 'back_key'
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
# 如果还在其他应用,启动目标应用
|
|
275
|
+
self.client.u2.app_start(self.target_package)
|
|
276
|
+
time.sleep(1)
|
|
277
|
+
|
|
278
|
+
# 验证是否成功返回
|
|
279
|
+
current = self._get_current_package()
|
|
280
|
+
if current == self.target_package:
|
|
281
|
+
return {
|
|
282
|
+
'success': True,
|
|
283
|
+
'message': f'✅ 已返回目标应用: {self.target_package}',
|
|
284
|
+
'method': 'app_start'
|
|
285
|
+
}
|
|
286
|
+
else:
|
|
287
|
+
return {
|
|
288
|
+
'success': False,
|
|
289
|
+
'message': f'❌ 返回失败:当前应用仍为 {current},期望 {self.target_package}',
|
|
290
|
+
'method': 'app_start'
|
|
291
|
+
}
|
|
292
|
+
except Exception as e:
|
|
293
|
+
return {
|
|
294
|
+
'success': False,
|
|
295
|
+
'message': f'❌ 返回目标应用失败: {e}',
|
|
296
|
+
'method': None
|
|
297
|
+
}
|
|
56
298
|
|
|
57
299
|
|
|
58
300
|
# ==================== 截图 ====================
|
|
@@ -107,7 +349,7 @@ class BasicMobileToolsLite:
|
|
|
107
349
|
size = ios_client.wda.window_size()
|
|
108
350
|
screen_width, screen_height = size[0], size[1]
|
|
109
351
|
else:
|
|
110
|
-
return {"success": False, "
|
|
352
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
111
353
|
else:
|
|
112
354
|
self.client.u2.screenshot(str(temp_path))
|
|
113
355
|
info = self.client.u2.info
|
|
@@ -158,22 +400,14 @@ class BasicMobileToolsLite:
|
|
|
158
400
|
|
|
159
401
|
cropped_size = final_path.stat().st_size
|
|
160
402
|
|
|
403
|
+
# 返回结果
|
|
161
404
|
return {
|
|
162
405
|
"success": True,
|
|
163
406
|
"screenshot_path": str(final_path),
|
|
164
|
-
"screen_width": screen_width,
|
|
165
|
-
"screen_height": screen_height,
|
|
166
407
|
"image_width": img.width,
|
|
167
408
|
"image_height": img.height,
|
|
168
409
|
"crop_offset_x": crop_offset_x,
|
|
169
|
-
"crop_offset_y": crop_offset_y
|
|
170
|
-
"file_size": f"{cropped_size/1024:.1f}KB",
|
|
171
|
-
"message": f"🔍 局部截图已保存: {final_path}\n"
|
|
172
|
-
f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
|
|
173
|
-
f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
|
|
174
|
-
f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
|
|
175
|
-
f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
|
|
176
|
-
f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
|
|
410
|
+
"crop_offset_y": crop_offset_y
|
|
177
411
|
}
|
|
178
412
|
|
|
179
413
|
# ========== 情况2:全屏压缩截图 ==========
|
|
@@ -226,24 +460,14 @@ class BasicMobileToolsLite:
|
|
|
226
460
|
compressed_size = final_path.stat().st_size
|
|
227
461
|
saved_percent = (1 - compressed_size / original_size) * 100
|
|
228
462
|
|
|
463
|
+
# 返回结果
|
|
229
464
|
return {
|
|
230
465
|
"success": True,
|
|
231
466
|
"screenshot_path": str(final_path),
|
|
232
|
-
"
|
|
233
|
-
"
|
|
234
|
-
"original_img_width": original_img_width,
|
|
235
|
-
"original_img_height": original_img_height
|
|
236
|
-
"image_width": image_width, # 压缩后宽度(AI 看到的)
|
|
237
|
-
"image_height": image_height, # 压缩后高度(AI 看到的)
|
|
238
|
-
"original_size": f"{original_size/1024:.1f}KB",
|
|
239
|
-
"compressed_size": f"{compressed_size/1024:.1f}KB",
|
|
240
|
-
"saved_percent": f"{saved_percent:.0f}%",
|
|
241
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
242
|
-
f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
|
|
243
|
-
f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
|
|
244
|
-
f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
|
|
245
|
-
f" image_width={image_width}, image_height={image_height},\n"
|
|
246
|
-
f" original_img_width={original_img_width}, original_img_height={original_img_height}"
|
|
467
|
+
"image_width": image_width,
|
|
468
|
+
"image_height": image_height,
|
|
469
|
+
"original_img_width": original_img_width,
|
|
470
|
+
"original_img_height": original_img_height
|
|
247
471
|
}
|
|
248
472
|
|
|
249
473
|
# ========== 情况3:全屏不压缩截图 ==========
|
|
@@ -257,21 +481,12 @@ class BasicMobileToolsLite:
|
|
|
257
481
|
final_path = self.screenshot_dir / filename
|
|
258
482
|
temp_path.rename(final_path)
|
|
259
483
|
|
|
260
|
-
#
|
|
484
|
+
# 返回结果(不压缩时尺寸相同)
|
|
261
485
|
return {
|
|
262
486
|
"success": True,
|
|
263
487
|
"screenshot_path": str(final_path),
|
|
264
|
-
"
|
|
265
|
-
"
|
|
266
|
-
"original_img_width": img.width, # 截图实际尺寸
|
|
267
|
-
"original_img_height": img.height,
|
|
268
|
-
"image_width": img.width, # 未压缩,和原图一样
|
|
269
|
-
"image_height": img.height,
|
|
270
|
-
"file_size": f"{original_size/1024:.1f}KB",
|
|
271
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
272
|
-
f"📐 截图尺寸: {img.width}x{img.height}\n"
|
|
273
|
-
f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
|
|
274
|
-
f"💡 未压缩,坐标可直接使用"
|
|
488
|
+
"image_width": img.width,
|
|
489
|
+
"image_height": img.height
|
|
275
490
|
}
|
|
276
491
|
except ImportError:
|
|
277
492
|
# 如果没有 PIL,回退到原始方式(不压缩)
|
|
@@ -279,7 +494,7 @@ class BasicMobileToolsLite:
|
|
|
279
494
|
except Exception as e:
|
|
280
495
|
return {"success": False, "message": f"❌ 截图失败: {e}"}
|
|
281
496
|
|
|
282
|
-
def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool =
|
|
497
|
+
def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool = False) -> Dict:
|
|
283
498
|
"""截图并添加网格坐标标注(用于精确定位元素)
|
|
284
499
|
|
|
285
500
|
在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
|
|
@@ -311,7 +526,7 @@ class BasicMobileToolsLite:
|
|
|
311
526
|
size = ios_client.wda.window_size()
|
|
312
527
|
screen_width, screen_height = size[0], size[1]
|
|
313
528
|
else:
|
|
314
|
-
return {"success": False, "
|
|
529
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
315
530
|
else:
|
|
316
531
|
self.client.u2.screenshot(str(temp_path))
|
|
317
532
|
info = self.client.u2.info
|
|
@@ -347,7 +562,7 @@ class BasicMobileToolsLite:
|
|
|
347
562
|
# 左侧标注 Y 坐标
|
|
348
563
|
draw.text((2, y + 2), str(y), fill=text_color, font=font_small)
|
|
349
564
|
|
|
350
|
-
# 第3
|
|
565
|
+
# 第3步:检测弹窗并标注(使用严格的置信度检测,避免误识别)
|
|
351
566
|
popup_info = None
|
|
352
567
|
close_positions = []
|
|
353
568
|
|
|
@@ -357,35 +572,12 @@ class BasicMobileToolsLite:
|
|
|
357
572
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
358
573
|
root = ET.fromstring(xml_string)
|
|
359
574
|
|
|
360
|
-
#
|
|
361
|
-
popup_bounds =
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
class_name = elem.attrib.get('class', '')
|
|
365
|
-
|
|
366
|
-
if not bounds_str:
|
|
367
|
-
continue
|
|
368
|
-
|
|
369
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
370
|
-
if not match:
|
|
371
|
-
continue
|
|
372
|
-
|
|
373
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
374
|
-
width = x2 - x1
|
|
375
|
-
height = y2 - y1
|
|
376
|
-
area = width * height
|
|
377
|
-
screen_area = screen_width * screen_height
|
|
378
|
-
|
|
379
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card'])
|
|
380
|
-
area_ratio = area / screen_area if screen_area > 0 else 0
|
|
381
|
-
is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
|
|
382
|
-
is_reasonable_size = 0.08 < area_ratio < 0.85
|
|
383
|
-
|
|
384
|
-
if is_container and is_not_fullscreen and is_reasonable_size and y1 > 50:
|
|
385
|
-
if popup_bounds is None or area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
386
|
-
popup_bounds = (x1, y1, x2, y2)
|
|
575
|
+
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
576
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
577
|
+
root, screen_width, screen_height
|
|
578
|
+
)
|
|
387
579
|
|
|
388
|
-
if popup_bounds:
|
|
580
|
+
if popup_bounds and popup_confidence >= 0.6:
|
|
389
581
|
px1, py1, px2, py2 = popup_bounds
|
|
390
582
|
popup_width = px2 - px1
|
|
391
583
|
popup_height = py2 - py1
|
|
@@ -448,26 +640,16 @@ class BasicMobileToolsLite:
|
|
|
448
640
|
result = {
|
|
449
641
|
"success": True,
|
|
450
642
|
"screenshot_path": str(final_path),
|
|
451
|
-
"screen_width": screen_width,
|
|
452
|
-
"screen_height": screen_height,
|
|
453
643
|
"image_width": img_width,
|
|
454
644
|
"image_height": img_height,
|
|
455
|
-
"grid_size": grid_size
|
|
456
|
-
"message": f"📸 网格截图已保存: {final_path}\n"
|
|
457
|
-
f"📐 尺寸: {img_width}x{img_height}\n"
|
|
458
|
-
f"📏 网格间距: {grid_size}px"
|
|
645
|
+
"grid_size": grid_size
|
|
459
646
|
}
|
|
460
647
|
|
|
461
648
|
if popup_info:
|
|
462
|
-
result["
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
|
|
467
|
-
for pos in close_positions:
|
|
468
|
-
result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
|
|
469
|
-
else:
|
|
470
|
-
result["popup_detected"] = False
|
|
649
|
+
result["popup"] = popup_info["bounds"]
|
|
650
|
+
# 只返回前3个最可能的关闭按钮位置
|
|
651
|
+
if close_positions:
|
|
652
|
+
result["close_hints"] = [(p['x'], p['y']) for p in close_positions[:3]]
|
|
471
653
|
|
|
472
654
|
return result
|
|
473
655
|
|
|
@@ -504,7 +686,7 @@ class BasicMobileToolsLite:
|
|
|
504
686
|
size = ios_client.wda.window_size()
|
|
505
687
|
screen_width, screen_height = size[0], size[1]
|
|
506
688
|
else:
|
|
507
|
-
return {"success": False, "
|
|
689
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
508
690
|
else:
|
|
509
691
|
self.client.u2.screenshot(str(temp_path))
|
|
510
692
|
info = self.client.u2.info
|
|
@@ -613,44 +795,24 @@ class BasicMobileToolsLite:
|
|
|
613
795
|
'index': i + 1,
|
|
614
796
|
'center': (cx, cy),
|
|
615
797
|
'bounds': f"[{x1},{y1}][{x2},{y2}]",
|
|
616
|
-
'desc': elem['desc']
|
|
798
|
+
'desc': elem['desc'],
|
|
799
|
+
'text': elem.get('text', ''),
|
|
800
|
+
'resource_id': elem.get('resource_id', '')
|
|
617
801
|
})
|
|
618
802
|
|
|
619
|
-
# 第3.5
|
|
803
|
+
# 第3.5步:检测弹窗区域(使用严格的置信度检测,避免误识别普通页面)
|
|
620
804
|
popup_bounds = None
|
|
805
|
+
popup_confidence = 0
|
|
621
806
|
|
|
622
807
|
if not self._is_ios():
|
|
623
808
|
try:
|
|
624
|
-
#
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
if not bounds_str:
|
|
630
|
-
continue
|
|
631
|
-
|
|
632
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
633
|
-
if not match:
|
|
634
|
-
continue
|
|
635
|
-
|
|
636
|
-
px1, py1, px2, py2 = map(int, match.groups())
|
|
637
|
-
p_width = px2 - px1
|
|
638
|
-
p_height = py2 - py1
|
|
639
|
-
p_area = p_width * p_height
|
|
640
|
-
screen_area = screen_width * screen_height
|
|
641
|
-
|
|
642
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Frame'])
|
|
643
|
-
area_ratio = p_area / screen_area if screen_area > 0 else 0
|
|
644
|
-
is_not_fullscreen = (p_width < screen_width * 0.99 or p_height < screen_height * 0.95)
|
|
645
|
-
# 放宽面积范围:5% - 95%
|
|
646
|
-
is_reasonable_size = 0.05 < area_ratio < 0.95
|
|
647
|
-
|
|
648
|
-
if is_container and is_not_fullscreen and is_reasonable_size and py1 > 30:
|
|
649
|
-
if popup_bounds is None or p_area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
650
|
-
popup_bounds = (px1, py1, px2, py2)
|
|
809
|
+
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
810
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
811
|
+
root, screen_width, screen_height
|
|
812
|
+
)
|
|
651
813
|
|
|
652
814
|
# 如果检测到弹窗,标注弹窗边界(不再猜测X按钮位置)
|
|
653
|
-
if popup_bounds:
|
|
815
|
+
if popup_bounds and popup_confidence >= 0.6:
|
|
654
816
|
px1, py1, px2, py2 = popup_bounds
|
|
655
817
|
|
|
656
818
|
# 只画弹窗边框(蓝色),不再猜测X按钮位置
|
|
@@ -684,38 +846,15 @@ class BasicMobileToolsLite:
|
|
|
684
846
|
img.save(str(final_path), "JPEG", quality=85)
|
|
685
847
|
temp_path.unlink()
|
|
686
848
|
|
|
687
|
-
#
|
|
688
|
-
elements_text = "\n".join([
|
|
689
|
-
f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
|
|
690
|
-
for e in som_elements[:15] # 只显示前15个
|
|
691
|
-
])
|
|
692
|
-
if len(som_elements) > 15:
|
|
693
|
-
elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
|
|
694
|
-
|
|
695
|
-
# 构建弹窗提示文字
|
|
696
|
-
hints_text = ""
|
|
697
|
-
if popup_bounds:
|
|
698
|
-
hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
|
|
699
|
-
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
700
|
-
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
701
|
-
|
|
849
|
+
# 返回结果(Token 优化:不返回 elements 列表,已存储在 self._som_elements)
|
|
702
850
|
return {
|
|
703
851
|
"success": True,
|
|
704
852
|
"screenshot_path": str(final_path),
|
|
705
853
|
"screen_width": screen_width,
|
|
706
854
|
"screen_height": screen_height,
|
|
707
|
-
"image_width": img_width,
|
|
708
|
-
"image_height": img_height,
|
|
709
855
|
"element_count": len(som_elements),
|
|
710
|
-
"elements": som_elements,
|
|
711
856
|
"popup_detected": popup_bounds is not None,
|
|
712
|
-
"
|
|
713
|
-
"message": f"📸 SoM 截图已保存: {final_path}\n"
|
|
714
|
-
f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
715
|
-
f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
716
|
-
f"💡 使用方法:\n"
|
|
717
|
-
f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
718
|
-
f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
857
|
+
"hint": "查看截图上的编号,用 click_by_som(编号) 点击"
|
|
719
858
|
}
|
|
720
859
|
|
|
721
860
|
except ImportError:
|
|
@@ -761,14 +900,41 @@ class BasicMobileToolsLite:
|
|
|
761
900
|
ios_client = self._get_ios_client()
|
|
762
901
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
763
902
|
ios_client.wda.click(cx, cy)
|
|
903
|
+
size = ios_client.wda.window_size()
|
|
904
|
+
screen_width, screen_height = size[0], size[1]
|
|
764
905
|
else:
|
|
765
906
|
self.client.u2.click(cx, cy)
|
|
766
|
-
|
|
907
|
+
info = self.client.u2.info
|
|
908
|
+
screen_width = info.get('displayWidth', 0)
|
|
909
|
+
screen_height = info.get('displayHeight', 0)
|
|
910
|
+
|
|
767
911
|
time.sleep(0.3)
|
|
768
912
|
|
|
913
|
+
# 计算百分比坐标用于跨设备兼容
|
|
914
|
+
x_percent = round(cx / screen_width * 100, 1) if screen_width > 0 else 0
|
|
915
|
+
y_percent = round(cy / screen_height * 100, 1) if screen_height > 0 else 0
|
|
916
|
+
|
|
917
|
+
# 使用标准记录格式
|
|
918
|
+
# 优先使用元素的文本/描述信息,这样生成脚本时可以用文本定位
|
|
919
|
+
elem_text = target.get('text', '')
|
|
920
|
+
elem_id = target.get('resource_id', '')
|
|
921
|
+
elem_desc = target.get('desc', '')
|
|
922
|
+
|
|
923
|
+
if elem_text and not elem_text.startswith('['): # 排除类似 "[可点击]" 的描述
|
|
924
|
+
# 有文本,使用文本定位
|
|
925
|
+
self._record_click('text', elem_text, x_percent, y_percent,
|
|
926
|
+
element_desc=f"[{index}]{elem_desc}", locator_attr='text')
|
|
927
|
+
elif elem_id:
|
|
928
|
+
# 有 resource-id,使用 ID 定位
|
|
929
|
+
self._record_click('id', elem_id, x_percent, y_percent,
|
|
930
|
+
element_desc=f"[{index}]{elem_desc}")
|
|
931
|
+
else:
|
|
932
|
+
# 都没有,使用百分比定位
|
|
933
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
934
|
+
element_desc=f"[{index}]{elem_desc}")
|
|
935
|
+
|
|
769
936
|
return {
|
|
770
937
|
"success": True,
|
|
771
|
-
"message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
|
|
772
938
|
"clicked": {
|
|
773
939
|
"index": index,
|
|
774
940
|
"desc": target['desc'],
|
|
@@ -802,7 +968,7 @@ class BasicMobileToolsLite:
|
|
|
802
968
|
size = ios_client.wda.window_size()
|
|
803
969
|
width, height = size[0], size[1]
|
|
804
970
|
else:
|
|
805
|
-
return {"success": False, "
|
|
971
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
806
972
|
else:
|
|
807
973
|
self.client.u2.screenshot(str(screenshot_path))
|
|
808
974
|
info = self.client.u2.info
|
|
@@ -880,7 +1046,7 @@ class BasicMobileToolsLite:
|
|
|
880
1046
|
size = ios_client.wda.window_size()
|
|
881
1047
|
screen_width, screen_height = size[0], size[1]
|
|
882
1048
|
else:
|
|
883
|
-
return {"success": False, "
|
|
1049
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
884
1050
|
else:
|
|
885
1051
|
info = self.client.u2.info
|
|
886
1052
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -924,37 +1090,45 @@ class BasicMobileToolsLite:
|
|
|
924
1090
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
925
1091
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
926
1092
|
|
|
927
|
-
#
|
|
928
|
-
self.
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1093
|
+
# 使用标准记录格式:坐标点击用百分比作为定位方式(跨分辨率兼容)
|
|
1094
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
1095
|
+
element_desc=f"坐标({x},{y})")
|
|
1096
|
+
|
|
1097
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1098
|
+
app_check = self._check_app_switched()
|
|
1099
|
+
return_result = None
|
|
1100
|
+
|
|
1101
|
+
if app_check['switched']:
|
|
1102
|
+
# 应用已跳转,尝试返回目标应用
|
|
1103
|
+
return_result = self._return_to_target_app()
|
|
938
1104
|
|
|
1105
|
+
# 构建返回消息
|
|
939
1106
|
if converted:
|
|
940
1107
|
if conversion_type == "crop_offset":
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
"message": f"✅ 点击成功: ({x}, {y})\n"
|
|
944
|
-
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
945
|
-
}
|
|
1108
|
+
msg = f"✅ 点击成功: ({x}, {y})\n" \
|
|
1109
|
+
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
946
1110
|
else:
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
951
|
-
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
952
|
-
}
|
|
1111
|
+
msg = f"✅ 点击成功: ({x}, {y})\n" \
|
|
1112
|
+
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n" \
|
|
1113
|
+
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
953
1114
|
else:
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
1115
|
+
msg = f"✅ 点击成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%]"
|
|
1116
|
+
|
|
1117
|
+
# 如果检测到应用跳转,添加警告和返回结果
|
|
1118
|
+
if app_check['switched']:
|
|
1119
|
+
msg += f"\n{app_check['message']}"
|
|
1120
|
+
if return_result:
|
|
1121
|
+
if return_result['success']:
|
|
1122
|
+
msg += f"\n{return_result['message']}"
|
|
1123
|
+
else:
|
|
1124
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1125
|
+
|
|
1126
|
+
return {
|
|
1127
|
+
"success": True,
|
|
1128
|
+
"message": msg,
|
|
1129
|
+
"app_check": app_check,
|
|
1130
|
+
"return_to_app": return_result
|
|
1131
|
+
}
|
|
958
1132
|
except Exception as e:
|
|
959
1133
|
return {"success": False, "message": f"❌ 点击失败: {e}"}
|
|
960
1134
|
|
|
@@ -987,14 +1161,14 @@ class BasicMobileToolsLite:
|
|
|
987
1161
|
size = ios_client.wda.window_size()
|
|
988
1162
|
width, height = size[0], size[1]
|
|
989
1163
|
else:
|
|
990
|
-
return {"success": False, "
|
|
1164
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
991
1165
|
else:
|
|
992
1166
|
info = self.client.u2.info
|
|
993
1167
|
width = info.get('displayWidth', 0)
|
|
994
1168
|
height = info.get('displayHeight', 0)
|
|
995
1169
|
|
|
996
1170
|
if width == 0 or height == 0:
|
|
997
|
-
return {"success": False, "
|
|
1171
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
998
1172
|
|
|
999
1173
|
# 第2步:百分比转像素坐标
|
|
1000
1174
|
# 公式:像素 = 屏幕尺寸 × (百分比 / 100)
|
|
@@ -1009,30 +1183,29 @@ class BasicMobileToolsLite:
|
|
|
1009
1183
|
|
|
1010
1184
|
time.sleep(0.3)
|
|
1011
1185
|
|
|
1012
|
-
# 第4
|
|
1013
|
-
self.
|
|
1014
|
-
|
|
1015
|
-
x=x,
|
|
1016
|
-
y=y,
|
|
1017
|
-
x_percent=x_percent,
|
|
1018
|
-
y_percent=y_percent,
|
|
1019
|
-
screen_width=width,
|
|
1020
|
-
screen_height=height,
|
|
1021
|
-
ref=f"percent_{x_percent}_{y_percent}"
|
|
1022
|
-
)
|
|
1186
|
+
# 第4步:使用标准记录格式
|
|
1187
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
1188
|
+
element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1023
1189
|
|
|
1024
1190
|
return {
|
|
1025
1191
|
"success": True,
|
|
1026
|
-
"message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
|
|
1027
|
-
"screen_size": {"width": width, "height": height},
|
|
1028
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1029
1192
|
"pixel": {"x": x, "y": y}
|
|
1030
1193
|
}
|
|
1031
1194
|
except Exception as e:
|
|
1032
1195
|
return {"success": False, "message": f"❌ 百分比点击失败: {e}"}
|
|
1033
1196
|
|
|
1034
|
-
def click_by_text(self, text: str, timeout: float = 3.0
|
|
1035
|
-
|
|
1197
|
+
def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None,
|
|
1198
|
+
verify: Optional[str] = None) -> Dict:
|
|
1199
|
+
"""通过文本点击 - 先查 XML 树,再精准匹配
|
|
1200
|
+
|
|
1201
|
+
Args:
|
|
1202
|
+
text: 元素的文本内容
|
|
1203
|
+
timeout: 超时时间
|
|
1204
|
+
position: 位置信息,当有多个相同文案时使用。支持:
|
|
1205
|
+
- 垂直方向: "top"/"upper"/"上", "bottom"/"lower"/"下", "middle"/"center"/"中"
|
|
1206
|
+
- 水平方向: "left"/"左", "right"/"右", "center"/"中"
|
|
1207
|
+
verify: 可选,点击后验证的文本。如果指定,会检查该文本是否出现在页面上
|
|
1208
|
+
"""
|
|
1036
1209
|
try:
|
|
1037
1210
|
if self._is_ios():
|
|
1038
1211
|
ios_client = self._get_ios_client()
|
|
@@ -1043,19 +1216,53 @@ class BasicMobileToolsLite:
|
|
|
1043
1216
|
if elem.exists:
|
|
1044
1217
|
elem.click()
|
|
1045
1218
|
time.sleep(0.3)
|
|
1046
|
-
self.
|
|
1047
|
-
|
|
1048
|
-
|
|
1219
|
+
self._record_click('text', text, element_desc=text, locator_attr='text')
|
|
1220
|
+
# 验证逻辑
|
|
1221
|
+
if verify:
|
|
1222
|
+
return self._verify_after_click(verify, ios=True)
|
|
1223
|
+
# 返回页面文本摘要,方便确认页面变化
|
|
1224
|
+
page_texts = self._get_page_texts(10)
|
|
1225
|
+
return {"success": True, "page_texts": page_texts}
|
|
1226
|
+
# 控件树找不到,提示用视觉识别
|
|
1227
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1228
|
+
else:
|
|
1229
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1049
1230
|
else:
|
|
1231
|
+
# 获取屏幕尺寸用于计算百分比
|
|
1232
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
1233
|
+
|
|
1050
1234
|
# 🔍 先查 XML 树,找到元素及其属性
|
|
1051
|
-
found_elem = self._find_element_in_tree(text)
|
|
1235
|
+
found_elem = self._find_element_in_tree(text, position=position)
|
|
1052
1236
|
|
|
1053
1237
|
if found_elem:
|
|
1054
1238
|
attr_type = found_elem['attr_type']
|
|
1055
1239
|
attr_value = found_elem['attr_value']
|
|
1056
1240
|
bounds = found_elem.get('bounds')
|
|
1057
1241
|
|
|
1058
|
-
#
|
|
1242
|
+
# 计算百分比坐标作为兜底
|
|
1243
|
+
x_pct, y_pct = 0, 0
|
|
1244
|
+
if bounds:
|
|
1245
|
+
cx = (bounds[0] + bounds[2]) // 2
|
|
1246
|
+
cy = (bounds[1] + bounds[3]) // 2
|
|
1247
|
+
x_pct = round(cx / screen_width * 100, 1)
|
|
1248
|
+
y_pct = round(cy / screen_height * 100, 1)
|
|
1249
|
+
|
|
1250
|
+
# 如果有位置参数,直接使用坐标点击
|
|
1251
|
+
if position and bounds:
|
|
1252
|
+
x = (bounds[0] + bounds[2]) // 2
|
|
1253
|
+
y = (bounds[1] + bounds[3]) // 2
|
|
1254
|
+
self.client.u2.click(x, y)
|
|
1255
|
+
time.sleep(0.3)
|
|
1256
|
+
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1257
|
+
element_desc=f"{text}({position})", locator_attr=attr_type)
|
|
1258
|
+
# 验证逻辑
|
|
1259
|
+
if verify:
|
|
1260
|
+
return self._verify_after_click(verify)
|
|
1261
|
+
# 返回页面文本摘要
|
|
1262
|
+
page_texts = self._get_page_texts(10)
|
|
1263
|
+
return {"success": True, "page_texts": page_texts}
|
|
1264
|
+
|
|
1265
|
+
# 没有位置参数时,使用选择器定位
|
|
1059
1266
|
if attr_type == 'text':
|
|
1060
1267
|
elem = self.client.u2(text=attr_value)
|
|
1061
1268
|
elif attr_type == 'textContains':
|
|
@@ -1070,33 +1277,98 @@ class BasicMobileToolsLite:
|
|
|
1070
1277
|
if elem and elem.exists(timeout=1):
|
|
1071
1278
|
elem.click()
|
|
1072
1279
|
time.sleep(0.3)
|
|
1073
|
-
self.
|
|
1074
|
-
|
|
1280
|
+
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1281
|
+
element_desc=text, locator_attr=attr_type)
|
|
1282
|
+
# 验证逻辑
|
|
1283
|
+
if verify:
|
|
1284
|
+
return self._verify_after_click(verify)
|
|
1285
|
+
# 返回页面文本摘要
|
|
1286
|
+
page_texts = self._get_page_texts(10)
|
|
1287
|
+
return {"success": True, "page_texts": page_texts}
|
|
1075
1288
|
|
|
1076
|
-
#
|
|
1289
|
+
# 选择器失败,用坐标兜底
|
|
1077
1290
|
if bounds:
|
|
1078
1291
|
x = (bounds[0] + bounds[2]) // 2
|
|
1079
1292
|
y = (bounds[1] + bounds[3]) // 2
|
|
1080
1293
|
self.client.u2.click(x, y)
|
|
1081
1294
|
time.sleep(0.3)
|
|
1082
|
-
self.
|
|
1083
|
-
|
|
1295
|
+
self._record_click('percent', f"{x_pct}%,{y_pct}%", x_pct, y_pct,
|
|
1296
|
+
element_desc=text)
|
|
1297
|
+
# 验证逻辑
|
|
1298
|
+
if verify:
|
|
1299
|
+
return self._verify_after_click(verify)
|
|
1300
|
+
# 返回页面文本摘要
|
|
1301
|
+
page_texts = self._get_page_texts(10)
|
|
1302
|
+
return {"success": True, "page_texts": page_texts}
|
|
1084
1303
|
|
|
1085
|
-
|
|
1304
|
+
# 控件树找不到,提示用视觉识别
|
|
1305
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1086
1306
|
except Exception as e:
|
|
1087
|
-
return {"success": False, "
|
|
1307
|
+
return {"success": False, "msg": str(e)}
|
|
1308
|
+
|
|
1309
|
+
def _verify_after_click(self, verify_text: str, ios: bool = False, timeout: float = 2.0) -> Dict:
|
|
1310
|
+
"""点击后验证期望文本是否出现
|
|
1311
|
+
|
|
1312
|
+
Args:
|
|
1313
|
+
verify_text: 期望出现的文本
|
|
1314
|
+
ios: 是否是 iOS 设备
|
|
1315
|
+
timeout: 验证超时时间
|
|
1316
|
+
|
|
1317
|
+
Returns:
|
|
1318
|
+
{"success": True, "verified": True/False, "hint": "..."}
|
|
1319
|
+
"""
|
|
1320
|
+
time.sleep(0.5) # 等待页面更新
|
|
1321
|
+
|
|
1322
|
+
try:
|
|
1323
|
+
if ios:
|
|
1324
|
+
ios_client = self._get_ios_client()
|
|
1325
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1326
|
+
exists = ios_client.wda(name=verify_text).exists or \
|
|
1327
|
+
ios_client.wda(label=verify_text).exists
|
|
1328
|
+
else:
|
|
1329
|
+
exists = False
|
|
1330
|
+
else:
|
|
1331
|
+
# Android: 检查文本或包含文本
|
|
1332
|
+
exists = self.client.u2(text=verify_text).exists(timeout=timeout) or \
|
|
1333
|
+
self.client.u2(textContains=verify_text).exists(timeout=0.5) or \
|
|
1334
|
+
self.client.u2(description=verify_text).exists(timeout=0.5)
|
|
1335
|
+
|
|
1336
|
+
if exists:
|
|
1337
|
+
return {"success": True, "verified": True}
|
|
1338
|
+
else:
|
|
1339
|
+
# 验证失败,提示可以截图确认
|
|
1340
|
+
return {
|
|
1341
|
+
"success": True, # 点击本身成功
|
|
1342
|
+
"verified": False,
|
|
1343
|
+
"expect": verify_text,
|
|
1344
|
+
"hint": "验证失败,可截图确认"
|
|
1345
|
+
}
|
|
1346
|
+
except Exception as e:
|
|
1347
|
+
return {"success": True, "verified": False, "hint": f"验证异常: {e}"}
|
|
1088
1348
|
|
|
1089
|
-
def _find_element_in_tree(self, text: str) -> Optional[Dict]:
|
|
1090
|
-
"""在 XML
|
|
1349
|
+
def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
|
|
1350
|
+
"""在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
|
|
1351
|
+
|
|
1352
|
+
Args:
|
|
1353
|
+
text: 要查找的文本
|
|
1354
|
+
position: 位置信息,用于在有多个相同文案时筛选
|
|
1355
|
+
"""
|
|
1091
1356
|
try:
|
|
1092
1357
|
xml = self.client.u2.dump_hierarchy(compressed=False)
|
|
1093
1358
|
import xml.etree.ElementTree as ET
|
|
1094
1359
|
root = ET.fromstring(xml)
|
|
1095
1360
|
|
|
1361
|
+
# 获取屏幕尺寸
|
|
1362
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
1363
|
+
|
|
1364
|
+
# 存储所有匹配的元素(包括不可点击的)
|
|
1365
|
+
matched_elements = []
|
|
1366
|
+
|
|
1096
1367
|
for elem in root.iter():
|
|
1097
1368
|
elem_text = elem.attrib.get('text', '')
|
|
1098
1369
|
elem_desc = elem.attrib.get('content-desc', '')
|
|
1099
1370
|
bounds_str = elem.attrib.get('bounds', '')
|
|
1371
|
+
clickable = elem.attrib.get('clickable', 'false').lower() == 'true'
|
|
1100
1372
|
|
|
1101
1373
|
# 解析 bounds
|
|
1102
1374
|
bounds = None
|
|
@@ -1106,36 +1378,113 @@ class BasicMobileToolsLite:
|
|
|
1106
1378
|
if len(match) == 4:
|
|
1107
1379
|
bounds = [int(x) for x in match]
|
|
1108
1380
|
|
|
1381
|
+
# 判断是否匹配
|
|
1382
|
+
is_match = False
|
|
1383
|
+
attr_type = None
|
|
1384
|
+
attr_value = None
|
|
1385
|
+
|
|
1109
1386
|
# 精确匹配 text
|
|
1110
1387
|
if elem_text == text:
|
|
1111
|
-
|
|
1112
|
-
|
|
1388
|
+
is_match = True
|
|
1389
|
+
attr_type = 'text'
|
|
1390
|
+
attr_value = text
|
|
1113
1391
|
# 精确匹配 content-desc
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1392
|
+
elif elem_desc == text:
|
|
1393
|
+
is_match = True
|
|
1394
|
+
attr_type = 'description'
|
|
1395
|
+
attr_value = text
|
|
1117
1396
|
# 模糊匹配 text
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1397
|
+
elif text in elem_text:
|
|
1398
|
+
is_match = True
|
|
1399
|
+
attr_type = 'textContains'
|
|
1400
|
+
attr_value = text
|
|
1121
1401
|
# 模糊匹配 content-desc
|
|
1122
|
-
|
|
1123
|
-
|
|
1402
|
+
elif text in elem_desc:
|
|
1403
|
+
is_match = True
|
|
1404
|
+
attr_type = 'descriptionContains'
|
|
1405
|
+
attr_value = text
|
|
1406
|
+
|
|
1407
|
+
if is_match and bounds:
|
|
1408
|
+
# 计算元素的中心点坐标
|
|
1409
|
+
center_x = (bounds[0] + bounds[2]) / 2
|
|
1410
|
+
center_y = (bounds[1] + bounds[3]) / 2
|
|
1411
|
+
|
|
1412
|
+
matched_elements.append({
|
|
1413
|
+
'attr_type': attr_type,
|
|
1414
|
+
'attr_value': attr_value,
|
|
1415
|
+
'bounds': bounds,
|
|
1416
|
+
'clickable': clickable,
|
|
1417
|
+
'center_x': center_x,
|
|
1418
|
+
'center_y': center_y
|
|
1419
|
+
})
|
|
1420
|
+
|
|
1421
|
+
if not matched_elements:
|
|
1422
|
+
return None
|
|
1423
|
+
|
|
1424
|
+
# 如果有位置信息,根据位置筛选
|
|
1425
|
+
if position and len(matched_elements) > 1:
|
|
1426
|
+
position_lower = position.lower()
|
|
1427
|
+
|
|
1428
|
+
# 根据位置信息排序
|
|
1429
|
+
if position_lower in ['top', 'upper', '上', '上方']:
|
|
1430
|
+
# 选择 y 坐标最小的(最上面的)
|
|
1431
|
+
matched_elements = sorted(matched_elements, key=lambda x: x['center_y'])
|
|
1432
|
+
elif position_lower in ['bottom', 'lower', '下', '下方', '底部']:
|
|
1433
|
+
# 选择 y 坐标最大的(最下面的)
|
|
1434
|
+
matched_elements = sorted(matched_elements, key=lambda x: x['center_y'], reverse=True)
|
|
1435
|
+
elif position_lower in ['left', '左', '左侧']:
|
|
1436
|
+
# 选择 x 坐标最小的(最左边的)
|
|
1437
|
+
matched_elements = sorted(matched_elements, key=lambda x: x['center_x'])
|
|
1438
|
+
elif position_lower in ['right', '右', '右侧']:
|
|
1439
|
+
# 选择 x 坐标最大的(最右边的)
|
|
1440
|
+
matched_elements = sorted(matched_elements, key=lambda x: x['center_x'], reverse=True)
|
|
1441
|
+
elif position_lower in ['middle', 'center', '中', '中间']:
|
|
1442
|
+
# 选择最接近屏幕中心的
|
|
1443
|
+
screen_mid_x = screen_width / 2
|
|
1444
|
+
screen_mid_y = screen_height / 2
|
|
1445
|
+
matched_elements = sorted(
|
|
1446
|
+
matched_elements,
|
|
1447
|
+
key=lambda x: abs(x['center_x'] - screen_mid_x) + abs(x['center_y'] - screen_mid_y)
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
# 如果有位置信息,优先返回排序后的第一个元素(最符合位置要求的)
|
|
1451
|
+
# 如果没有位置信息,优先返回可点击的元素
|
|
1452
|
+
if position and matched_elements:
|
|
1453
|
+
# 有位置信息时,直接返回排序后的第一个(最符合位置要求的)
|
|
1454
|
+
first_match = matched_elements[0]
|
|
1455
|
+
return {
|
|
1456
|
+
'attr_type': first_match['attr_type'],
|
|
1457
|
+
'attr_value': first_match['attr_value'],
|
|
1458
|
+
'bounds': first_match['bounds']
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
# 没有位置信息时,优先返回可点击的元素
|
|
1462
|
+
for match in matched_elements:
|
|
1463
|
+
if match['clickable']:
|
|
1464
|
+
return {
|
|
1465
|
+
'attr_type': match['attr_type'],
|
|
1466
|
+
'attr_value': match['attr_value'],
|
|
1467
|
+
'bounds': match['bounds']
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
# 如果没有可点击的元素,直接返回第一个匹配元素的 bounds(使用坐标点击)
|
|
1471
|
+
if matched_elements:
|
|
1472
|
+
first_match = matched_elements[0]
|
|
1473
|
+
return {
|
|
1474
|
+
'attr_type': first_match['attr_type'],
|
|
1475
|
+
'attr_value': first_match['attr_value'],
|
|
1476
|
+
'bounds': first_match['bounds']
|
|
1477
|
+
}
|
|
1124
1478
|
|
|
1125
1479
|
return None
|
|
1126
|
-
except Exception:
|
|
1480
|
+
except Exception as e:
|
|
1481
|
+
import traceback
|
|
1482
|
+
traceback.print_exc()
|
|
1127
1483
|
return None
|
|
1128
1484
|
|
|
1129
1485
|
def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
|
|
1130
|
-
"""通过 resource-id
|
|
1131
|
-
|
|
1132
|
-
Args:
|
|
1133
|
-
resource_id: 元素的 resource-id
|
|
1134
|
-
index: 第几个元素(从 0 开始),默认 0 表示第一个
|
|
1135
|
-
"""
|
|
1486
|
+
"""通过 resource-id 点击"""
|
|
1136
1487
|
try:
|
|
1137
|
-
index_desc = f"[{index}]" if index > 0 else ""
|
|
1138
|
-
|
|
1139
1488
|
if self._is_ios():
|
|
1140
1489
|
ios_client = self._get_ios_client()
|
|
1141
1490
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
@@ -1143,31 +1492,31 @@ class BasicMobileToolsLite:
|
|
|
1143
1492
|
if not elem.exists:
|
|
1144
1493
|
elem = ios_client.wda(name=resource_id)
|
|
1145
1494
|
if elem.exists:
|
|
1146
|
-
# 获取所有匹配的元素
|
|
1147
1495
|
elements = elem.find_elements()
|
|
1148
1496
|
if index < len(elements):
|
|
1149
1497
|
elements[index].click()
|
|
1150
1498
|
time.sleep(0.3)
|
|
1151
|
-
self.
|
|
1152
|
-
return {"success": True
|
|
1499
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1500
|
+
return {"success": True}
|
|
1153
1501
|
else:
|
|
1154
|
-
return {"success": False, "
|
|
1155
|
-
return {"success": False, "
|
|
1502
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{len(elements)}个)"}
|
|
1503
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1504
|
+
else:
|
|
1505
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1156
1506
|
else:
|
|
1157
1507
|
elem = self.client.u2(resourceId=resource_id)
|
|
1158
1508
|
if elem.exists(timeout=0.5):
|
|
1159
|
-
# 获取匹配元素数量
|
|
1160
1509
|
count = elem.count
|
|
1161
1510
|
if index < count:
|
|
1162
1511
|
elem[index].click()
|
|
1163
1512
|
time.sleep(0.3)
|
|
1164
|
-
self.
|
|
1165
|
-
return {"success": True
|
|
1513
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1514
|
+
return {"success": True}
|
|
1166
1515
|
else:
|
|
1167
|
-
return {"success": False, "
|
|
1168
|
-
return {"success": False, "
|
|
1516
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{count}个)"}
|
|
1517
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1169
1518
|
except Exception as e:
|
|
1170
|
-
return {"success": False, "
|
|
1519
|
+
return {"success": False, "msg": str(e)}
|
|
1171
1520
|
|
|
1172
1521
|
# ==================== 长按操作 ====================
|
|
1173
1522
|
|
|
@@ -1201,7 +1550,7 @@ class BasicMobileToolsLite:
|
|
|
1201
1550
|
size = ios_client.wda.window_size()
|
|
1202
1551
|
screen_width, screen_height = size[0], size[1]
|
|
1203
1552
|
else:
|
|
1204
|
-
return {"success": False, "
|
|
1553
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1205
1554
|
else:
|
|
1206
1555
|
info = self.client.u2.info
|
|
1207
1556
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1248,38 +1597,17 @@ class BasicMobileToolsLite:
|
|
|
1248
1597
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1249
1598
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1250
1599
|
|
|
1251
|
-
#
|
|
1252
|
-
self.
|
|
1253
|
-
|
|
1254
|
-
x=x,
|
|
1255
|
-
y=y,
|
|
1256
|
-
x_percent=x_percent,
|
|
1257
|
-
y_percent=y_percent,
|
|
1258
|
-
duration=duration,
|
|
1259
|
-
screen_width=screen_width,
|
|
1260
|
-
screen_height=screen_height,
|
|
1261
|
-
ref=f"coords_{x}_{y}"
|
|
1262
|
-
)
|
|
1600
|
+
# 使用标准记录格式
|
|
1601
|
+
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1602
|
+
x_percent, y_percent, element_desc=f"坐标({x},{y})")
|
|
1263
1603
|
|
|
1264
1604
|
if converted:
|
|
1265
1605
|
if conversion_type == "crop_offset":
|
|
1266
|
-
return {
|
|
1267
|
-
"success": True,
|
|
1268
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1269
|
-
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
1270
|
-
}
|
|
1606
|
+
return {"success": True}
|
|
1271
1607
|
else:
|
|
1272
|
-
return {
|
|
1273
|
-
"success": True,
|
|
1274
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1275
|
-
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
1276
|
-
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
1277
|
-
}
|
|
1608
|
+
return {"success": True}
|
|
1278
1609
|
else:
|
|
1279
|
-
return {
|
|
1280
|
-
"success": True,
|
|
1281
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
|
|
1282
|
-
}
|
|
1610
|
+
return {"success": True}
|
|
1283
1611
|
except Exception as e:
|
|
1284
1612
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1285
1613
|
|
|
@@ -1308,14 +1636,14 @@ class BasicMobileToolsLite:
|
|
|
1308
1636
|
size = ios_client.wda.window_size()
|
|
1309
1637
|
width, height = size[0], size[1]
|
|
1310
1638
|
else:
|
|
1311
|
-
return {"success": False, "
|
|
1639
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1312
1640
|
else:
|
|
1313
1641
|
info = self.client.u2.info
|
|
1314
1642
|
width = info.get('displayWidth', 0)
|
|
1315
1643
|
height = info.get('displayHeight', 0)
|
|
1316
1644
|
|
|
1317
1645
|
if width == 0 or height == 0:
|
|
1318
|
-
return {"success": False, "
|
|
1646
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1319
1647
|
|
|
1320
1648
|
# 第2步:百分比转像素坐标
|
|
1321
1649
|
x = int(width * x_percent / 100)
|
|
@@ -1333,26 +1661,11 @@ class BasicMobileToolsLite:
|
|
|
1333
1661
|
|
|
1334
1662
|
time.sleep(0.3)
|
|
1335
1663
|
|
|
1336
|
-
# 第4
|
|
1337
|
-
self.
|
|
1338
|
-
|
|
1339
|
-
x=x,
|
|
1340
|
-
y=y,
|
|
1341
|
-
x_percent=x_percent,
|
|
1342
|
-
y_percent=y_percent,
|
|
1343
|
-
duration=duration,
|
|
1344
|
-
screen_width=width,
|
|
1345
|
-
screen_height=height,
|
|
1346
|
-
ref=f"percent_{x_percent}_{y_percent}"
|
|
1347
|
-
)
|
|
1664
|
+
# 第4步:使用标准记录格式
|
|
1665
|
+
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1666
|
+
x_percent, y_percent, element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1348
1667
|
|
|
1349
|
-
return {
|
|
1350
|
-
"success": True,
|
|
1351
|
-
"message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
|
|
1352
|
-
"screen_size": {"width": width, "height": height},
|
|
1353
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1354
|
-
"pixel": {"x": x, "y": y},
|
|
1355
|
-
"duration": duration
|
|
1668
|
+
return {"success": True
|
|
1356
1669
|
}
|
|
1357
1670
|
except Exception as e:
|
|
1358
1671
|
return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
|
|
@@ -1381,10 +1694,13 @@ class BasicMobileToolsLite:
|
|
|
1381
1694
|
else:
|
|
1382
1695
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1383
1696
|
time.sleep(0.3)
|
|
1384
|
-
self.
|
|
1385
|
-
return {"success": True
|
|
1386
|
-
return {"success": False, "
|
|
1697
|
+
self._record_long_press('text', text, duration, element_desc=text, locator_attr='text')
|
|
1698
|
+
return {"success": True}
|
|
1699
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1387
1700
|
else:
|
|
1701
|
+
# 获取屏幕尺寸用于计算百分比
|
|
1702
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
1703
|
+
|
|
1388
1704
|
# 先查 XML 树,找到元素
|
|
1389
1705
|
found_elem = self._find_element_in_tree(text)
|
|
1390
1706
|
|
|
@@ -1393,6 +1709,14 @@ class BasicMobileToolsLite:
|
|
|
1393
1709
|
attr_value = found_elem['attr_value']
|
|
1394
1710
|
bounds = found_elem.get('bounds')
|
|
1395
1711
|
|
|
1712
|
+
# 计算百分比坐标作为兜底
|
|
1713
|
+
x_pct, y_pct = 0, 0
|
|
1714
|
+
if bounds:
|
|
1715
|
+
cx = (bounds[0] + bounds[2]) // 2
|
|
1716
|
+
cy = (bounds[1] + bounds[3]) // 2
|
|
1717
|
+
x_pct = round(cx / screen_width * 100, 1)
|
|
1718
|
+
y_pct = round(cy / screen_height * 100, 1)
|
|
1719
|
+
|
|
1396
1720
|
# 根据找到的属性类型,使用对应的选择器
|
|
1397
1721
|
if attr_type == 'text':
|
|
1398
1722
|
elem = self.client.u2(text=attr_value)
|
|
@@ -1408,8 +1732,9 @@ class BasicMobileToolsLite:
|
|
|
1408
1732
|
if elem and elem.exists(timeout=1):
|
|
1409
1733
|
elem.long_click(duration=duration)
|
|
1410
1734
|
time.sleep(0.3)
|
|
1411
|
-
self.
|
|
1412
|
-
|
|
1735
|
+
self._record_long_press('text', attr_value, duration, x_pct, y_pct,
|
|
1736
|
+
element_desc=text, locator_attr=attr_type)
|
|
1737
|
+
return {"success": True}
|
|
1413
1738
|
|
|
1414
1739
|
# 如果选择器失败,用坐标兜底
|
|
1415
1740
|
if bounds:
|
|
@@ -1417,10 +1742,11 @@ class BasicMobileToolsLite:
|
|
|
1417
1742
|
y = (bounds[1] + bounds[3]) // 2
|
|
1418
1743
|
self.client.u2.long_click(x, y, duration=duration)
|
|
1419
1744
|
time.sleep(0.3)
|
|
1420
|
-
self.
|
|
1421
|
-
|
|
1745
|
+
self._record_long_press('percent', f"{x_pct}%,{y_pct}%", duration, x_pct, y_pct,
|
|
1746
|
+
element_desc=text)
|
|
1747
|
+
return {"success": True}
|
|
1422
1748
|
|
|
1423
|
-
return {"success": False, "
|
|
1749
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1424
1750
|
except Exception as e:
|
|
1425
1751
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1426
1752
|
|
|
@@ -1447,17 +1773,17 @@ class BasicMobileToolsLite:
|
|
|
1447
1773
|
else:
|
|
1448
1774
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1449
1775
|
time.sleep(0.3)
|
|
1450
|
-
self.
|
|
1451
|
-
return {"success": True
|
|
1452
|
-
return {"success": False, "
|
|
1776
|
+
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1777
|
+
return {"success": True}
|
|
1778
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1453
1779
|
else:
|
|
1454
1780
|
elem = self.client.u2(resourceId=resource_id)
|
|
1455
1781
|
if elem.exists(timeout=0.5):
|
|
1456
1782
|
elem.long_click(duration=duration)
|
|
1457
1783
|
time.sleep(0.3)
|
|
1458
|
-
self.
|
|
1784
|
+
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1459
1785
|
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
1460
|
-
return {"success": False, "
|
|
1786
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1461
1787
|
except Exception as e:
|
|
1462
1788
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1463
1789
|
|
|
@@ -1482,8 +1808,29 @@ class BasicMobileToolsLite:
|
|
|
1482
1808
|
if elem.exists:
|
|
1483
1809
|
elem.set_text(text)
|
|
1484
1810
|
time.sleep(0.3)
|
|
1485
|
-
self.
|
|
1486
|
-
|
|
1811
|
+
self._record_input(text, 'id', resource_id)
|
|
1812
|
+
|
|
1813
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1814
|
+
app_check = self._check_app_switched()
|
|
1815
|
+
return_result = None
|
|
1816
|
+
if app_check['switched']:
|
|
1817
|
+
return_result = self._return_to_target_app()
|
|
1818
|
+
|
|
1819
|
+
msg = f"✅ 输入成功: '{text}'"
|
|
1820
|
+
if app_check['switched']:
|
|
1821
|
+
msg += f"\n{app_check['message']}"
|
|
1822
|
+
if return_result:
|
|
1823
|
+
if return_result['success']:
|
|
1824
|
+
msg += f"\n{return_result['message']}"
|
|
1825
|
+
else:
|
|
1826
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1827
|
+
|
|
1828
|
+
return {
|
|
1829
|
+
"success": True,
|
|
1830
|
+
"message": msg,
|
|
1831
|
+
"app_check": app_check,
|
|
1832
|
+
"return_to_app": return_result
|
|
1833
|
+
}
|
|
1487
1834
|
return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
|
|
1488
1835
|
else:
|
|
1489
1836
|
elements = self.client.u2(resourceId=resource_id)
|
|
@@ -1496,8 +1843,29 @@ class BasicMobileToolsLite:
|
|
|
1496
1843
|
if count == 1:
|
|
1497
1844
|
elements.set_text(text)
|
|
1498
1845
|
time.sleep(0.3)
|
|
1499
|
-
self.
|
|
1500
|
-
|
|
1846
|
+
self._record_input(text, 'id', resource_id)
|
|
1847
|
+
|
|
1848
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1849
|
+
app_check = self._check_app_switched()
|
|
1850
|
+
return_result = None
|
|
1851
|
+
if app_check['switched']:
|
|
1852
|
+
return_result = self._return_to_target_app()
|
|
1853
|
+
|
|
1854
|
+
msg = f"✅ 输入成功: '{text}'"
|
|
1855
|
+
if app_check['switched']:
|
|
1856
|
+
msg += f"\n{app_check['message']}"
|
|
1857
|
+
if return_result:
|
|
1858
|
+
if return_result['success']:
|
|
1859
|
+
msg += f"\n{return_result['message']}"
|
|
1860
|
+
else:
|
|
1861
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1862
|
+
|
|
1863
|
+
return {
|
|
1864
|
+
"success": True,
|
|
1865
|
+
"message": msg,
|
|
1866
|
+
"app_check": app_check,
|
|
1867
|
+
"return_to_app": return_result
|
|
1868
|
+
}
|
|
1501
1869
|
|
|
1502
1870
|
# 多个相同 ID(<=5个),尝试智能选择
|
|
1503
1871
|
if count <= 5:
|
|
@@ -1509,15 +1877,57 @@ class BasicMobileToolsLite:
|
|
|
1509
1877
|
if info.get('editable') or info.get('focusable'):
|
|
1510
1878
|
elem.set_text(text)
|
|
1511
1879
|
time.sleep(0.3)
|
|
1512
|
-
self.
|
|
1513
|
-
|
|
1880
|
+
self._record_input(text, 'id', resource_id)
|
|
1881
|
+
|
|
1882
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1883
|
+
app_check = self._check_app_switched()
|
|
1884
|
+
return_result = None
|
|
1885
|
+
if app_check['switched']:
|
|
1886
|
+
return_result = self._return_to_target_app()
|
|
1887
|
+
|
|
1888
|
+
msg = f"✅ 输入成功: '{text}'"
|
|
1889
|
+
if app_check['switched']:
|
|
1890
|
+
msg += f"\n{app_check['message']}"
|
|
1891
|
+
if return_result:
|
|
1892
|
+
if return_result['success']:
|
|
1893
|
+
msg += f"\n{return_result['message']}"
|
|
1894
|
+
else:
|
|
1895
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1896
|
+
|
|
1897
|
+
return {
|
|
1898
|
+
"success": True,
|
|
1899
|
+
"message": msg,
|
|
1900
|
+
"app_check": app_check,
|
|
1901
|
+
"return_to_app": return_result
|
|
1902
|
+
}
|
|
1514
1903
|
except:
|
|
1515
1904
|
continue
|
|
1516
1905
|
# 没找到可编辑的,用第一个
|
|
1517
1906
|
elements[0].set_text(text)
|
|
1518
1907
|
time.sleep(0.3)
|
|
1519
|
-
self.
|
|
1520
|
-
|
|
1908
|
+
self._record_input(text, 'id', resource_id)
|
|
1909
|
+
|
|
1910
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1911
|
+
app_check = self._check_app_switched()
|
|
1912
|
+
return_result = None
|
|
1913
|
+
if app_check['switched']:
|
|
1914
|
+
return_result = self._return_to_target_app()
|
|
1915
|
+
|
|
1916
|
+
msg = f"✅ 输入成功: '{text}'"
|
|
1917
|
+
if app_check['switched']:
|
|
1918
|
+
msg += f"\n{app_check['message']}"
|
|
1919
|
+
if return_result:
|
|
1920
|
+
if return_result['success']:
|
|
1921
|
+
msg += f"\n{return_result['message']}"
|
|
1922
|
+
else:
|
|
1923
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1924
|
+
|
|
1925
|
+
return {
|
|
1926
|
+
"success": True,
|
|
1927
|
+
"message": msg,
|
|
1928
|
+
"app_check": app_check,
|
|
1929
|
+
"return_to_app": return_result
|
|
1930
|
+
}
|
|
1521
1931
|
|
|
1522
1932
|
# ID 不可靠(不存在或太多),改用 EditText 类型定位
|
|
1523
1933
|
edit_texts = self.client.u2(className='android.widget.EditText')
|
|
@@ -1526,10 +1936,31 @@ class BasicMobileToolsLite:
|
|
|
1526
1936
|
if et_count == 1:
|
|
1527
1937
|
edit_texts.set_text(text)
|
|
1528
1938
|
time.sleep(0.3)
|
|
1529
|
-
self.
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1939
|
+
self._record_input(text, 'class', 'EditText')
|
|
1940
|
+
|
|
1941
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1942
|
+
app_check = self._check_app_switched()
|
|
1943
|
+
return_result = None
|
|
1944
|
+
if app_check['switched']:
|
|
1945
|
+
return_result = self._return_to_target_app()
|
|
1946
|
+
|
|
1947
|
+
msg = f"✅ 输入成功: '{text}' (通过 EditText 定位)"
|
|
1948
|
+
if app_check['switched']:
|
|
1949
|
+
msg += f"\n{app_check['message']}"
|
|
1950
|
+
if return_result:
|
|
1951
|
+
if return_result['success']:
|
|
1952
|
+
msg += f"\n{return_result['message']}"
|
|
1953
|
+
else:
|
|
1954
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1955
|
+
|
|
1956
|
+
return {
|
|
1957
|
+
"success": True,
|
|
1958
|
+
"message": msg,
|
|
1959
|
+
"app_check": app_check,
|
|
1960
|
+
"return_to_app": return_result
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
# 多个 EditText,选择最靠上的
|
|
1533
1964
|
best_elem = None
|
|
1534
1965
|
min_top = 9999
|
|
1535
1966
|
for i in range(et_count):
|
|
@@ -1545,8 +1976,29 @@ class BasicMobileToolsLite:
|
|
|
1545
1976
|
if best_elem:
|
|
1546
1977
|
best_elem.set_text(text)
|
|
1547
1978
|
time.sleep(0.3)
|
|
1548
|
-
self.
|
|
1549
|
-
|
|
1979
|
+
self._record_input(text, 'class', 'EditText')
|
|
1980
|
+
|
|
1981
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1982
|
+
app_check = self._check_app_switched()
|
|
1983
|
+
return_result = None
|
|
1984
|
+
if app_check['switched']:
|
|
1985
|
+
return_result = self._return_to_target_app()
|
|
1986
|
+
|
|
1987
|
+
msg = f"✅ 输入成功: '{text}' (通过 EditText 定位,选择最顶部的)"
|
|
1988
|
+
if app_check['switched']:
|
|
1989
|
+
msg += f"\n{app_check['message']}"
|
|
1990
|
+
if return_result:
|
|
1991
|
+
if return_result['success']:
|
|
1992
|
+
msg += f"\n{return_result['message']}"
|
|
1993
|
+
else:
|
|
1994
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
1995
|
+
|
|
1996
|
+
return {
|
|
1997
|
+
"success": True,
|
|
1998
|
+
"message": msg,
|
|
1999
|
+
"app_check": app_check,
|
|
2000
|
+
"return_to_app": return_result
|
|
2001
|
+
}
|
|
1550
2002
|
|
|
1551
2003
|
return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
|
|
1552
2004
|
|
|
@@ -1588,17 +2040,32 @@ class BasicMobileToolsLite:
|
|
|
1588
2040
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1589
2041
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1590
2042
|
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
x=x,
|
|
1594
|
-
y=y,
|
|
1595
|
-
x_percent=x_percent,
|
|
1596
|
-
y_percent=y_percent,
|
|
1597
|
-
ref=f"coords_{x}_{y}",
|
|
1598
|
-
text=text
|
|
1599
|
-
)
|
|
2043
|
+
# 使用标准记录格式
|
|
2044
|
+
self._record_input(text, 'percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent)
|
|
1600
2045
|
|
|
1601
|
-
|
|
2046
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2047
|
+
app_check = self._check_app_switched()
|
|
2048
|
+
return_result = None
|
|
2049
|
+
|
|
2050
|
+
if app_check['switched']:
|
|
2051
|
+
# 应用已跳转,尝试返回目标应用
|
|
2052
|
+
return_result = self._return_to_target_app()
|
|
2053
|
+
|
|
2054
|
+
msg = f"✅ 输入成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%] -> '{text}'"
|
|
2055
|
+
if app_check['switched']:
|
|
2056
|
+
msg += f"\n{app_check['message']}"
|
|
2057
|
+
if return_result:
|
|
2058
|
+
if return_result['success']:
|
|
2059
|
+
msg += f"\n{return_result['message']}"
|
|
2060
|
+
else:
|
|
2061
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2062
|
+
|
|
2063
|
+
return {
|
|
2064
|
+
"success": True,
|
|
2065
|
+
"message": msg,
|
|
2066
|
+
"app_check": app_check,
|
|
2067
|
+
"return_to_app": return_result
|
|
2068
|
+
}
|
|
1602
2069
|
except Exception as e:
|
|
1603
2070
|
return {"success": False, "message": f"❌ 输入失败: {e}"}
|
|
1604
2071
|
|
|
@@ -1619,7 +2086,7 @@ class BasicMobileToolsLite:
|
|
|
1619
2086
|
size = ios_client.wda.window_size()
|
|
1620
2087
|
width, height = size[0], size[1]
|
|
1621
2088
|
else:
|
|
1622
|
-
return {"success": False, "
|
|
2089
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1623
2090
|
else:
|
|
1624
2091
|
width, height = self.client.u2.window_size()
|
|
1625
2092
|
|
|
@@ -1657,13 +2124,16 @@ class BasicMobileToolsLite:
|
|
|
1657
2124
|
else:
|
|
1658
2125
|
self.client.u2.swipe(x1, y1, x2, y2, duration=0.5)
|
|
1659
2126
|
|
|
1660
|
-
#
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
2127
|
+
# 使用标准记录格式
|
|
2128
|
+
self._record_swipe(direction)
|
|
2129
|
+
|
|
2130
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2131
|
+
app_check = self._check_app_switched()
|
|
2132
|
+
return_result = None
|
|
2133
|
+
|
|
2134
|
+
if app_check['switched']:
|
|
2135
|
+
# 应用已跳转,尝试返回目标应用
|
|
2136
|
+
return_result = self._return_to_target_app()
|
|
1667
2137
|
|
|
1668
2138
|
# 构建返回消息
|
|
1669
2139
|
msg = f"✅ 滑动成功: {direction}"
|
|
@@ -1673,7 +2143,21 @@ class BasicMobileToolsLite:
|
|
|
1673
2143
|
elif y is not None:
|
|
1674
2144
|
msg += f" (高度: {y}px)"
|
|
1675
2145
|
|
|
1676
|
-
|
|
2146
|
+
# 如果检测到应用跳转,添加警告和返回结果
|
|
2147
|
+
if app_check['switched']:
|
|
2148
|
+
msg += f"\n{app_check['message']}"
|
|
2149
|
+
if return_result:
|
|
2150
|
+
if return_result['success']:
|
|
2151
|
+
msg += f"\n{return_result['message']}"
|
|
2152
|
+
else:
|
|
2153
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2154
|
+
|
|
2155
|
+
return {
|
|
2156
|
+
"success": True,
|
|
2157
|
+
"message": msg,
|
|
2158
|
+
"app_check": app_check,
|
|
2159
|
+
"return_to_app": return_result
|
|
2160
|
+
}
|
|
1677
2161
|
except Exception as e:
|
|
1678
2162
|
return {"success": False, "message": f"❌ 滑动失败: {e}"}
|
|
1679
2163
|
|
|
@@ -1698,22 +2182,22 @@ class BasicMobileToolsLite:
|
|
|
1698
2182
|
ios_client.wda.send_keys('\n')
|
|
1699
2183
|
elif ios_key == 'home':
|
|
1700
2184
|
ios_client.wda.home()
|
|
1701
|
-
return {"success": True
|
|
1702
|
-
return {"success": False, "
|
|
2185
|
+
return {"success": True}
|
|
2186
|
+
return {"success": False, "msg": f"iOS不支持{key}"}
|
|
1703
2187
|
else:
|
|
1704
2188
|
keycode = key_map.get(key.lower())
|
|
1705
2189
|
if keycode:
|
|
1706
2190
|
self.client.u2.shell(f'input keyevent {keycode}')
|
|
1707
|
-
self.
|
|
1708
|
-
return {"success": True
|
|
1709
|
-
return {"success": False, "
|
|
2191
|
+
self._record_key(key)
|
|
2192
|
+
return {"success": True}
|
|
2193
|
+
return {"success": False, "msg": f"不支持按键{key}"}
|
|
1710
2194
|
except Exception as e:
|
|
1711
2195
|
return {"success": False, "message": f"❌ 按键失败: {e}"}
|
|
1712
2196
|
|
|
1713
2197
|
def wait(self, seconds: float) -> Dict:
|
|
1714
2198
|
"""等待指定时间"""
|
|
1715
2199
|
time.sleep(seconds)
|
|
1716
|
-
return {"success": True
|
|
2200
|
+
return {"success": True}
|
|
1717
2201
|
|
|
1718
2202
|
# ==================== 应用管理 ====================
|
|
1719
2203
|
|
|
@@ -1729,12 +2213,20 @@ class BasicMobileToolsLite:
|
|
|
1729
2213
|
|
|
1730
2214
|
await asyncio.sleep(2)
|
|
1731
2215
|
|
|
2216
|
+
# 记录目标应用包名(用于后续监测应用跳转)
|
|
2217
|
+
self.target_package = package_name
|
|
2218
|
+
|
|
2219
|
+
# 验证是否成功启动到目标应用
|
|
2220
|
+
current = self._get_current_package()
|
|
2221
|
+
if current and current != package_name:
|
|
2222
|
+
return {
|
|
2223
|
+
"success": False,
|
|
2224
|
+
"message": f"❌ 启动失败:当前应用为 {current},期望 {package_name}"
|
|
2225
|
+
}
|
|
2226
|
+
|
|
1732
2227
|
self._record_operation('launch_app', package_name=package_name)
|
|
1733
2228
|
|
|
1734
|
-
return {
|
|
1735
|
-
"success": True,
|
|
1736
|
-
"message": f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载"
|
|
1737
|
-
}
|
|
2229
|
+
return {"success": True}
|
|
1738
2230
|
except Exception as e:
|
|
1739
2231
|
return {"success": False, "message": f"❌ 启动失败: {e}"}
|
|
1740
2232
|
|
|
@@ -1747,9 +2239,9 @@ class BasicMobileToolsLite:
|
|
|
1747
2239
|
ios_client.wda.app_terminate(package_name)
|
|
1748
2240
|
else:
|
|
1749
2241
|
self.client.u2.app_stop(package_name)
|
|
1750
|
-
return {"success": True
|
|
2242
|
+
return {"success": True}
|
|
1751
2243
|
except Exception as e:
|
|
1752
|
-
return {"success": False, "
|
|
2244
|
+
return {"success": False, "msg": str(e)}
|
|
1753
2245
|
|
|
1754
2246
|
def list_apps(self, filter_keyword: str = "") -> Dict:
|
|
1755
2247
|
"""列出已安装应用"""
|
|
@@ -1823,7 +2315,7 @@ class BasicMobileToolsLite:
|
|
|
1823
2315
|
# ==================== 辅助工具 ====================
|
|
1824
2316
|
|
|
1825
2317
|
def list_elements(self) -> List[Dict]:
|
|
1826
|
-
"""
|
|
2318
|
+
"""列出页面元素(已优化:过滤排版容器,保留功能控件)"""
|
|
1827
2319
|
try:
|
|
1828
2320
|
if self._is_ios():
|
|
1829
2321
|
ios_client = self._get_ios_client()
|
|
@@ -1834,20 +2326,268 @@ class BasicMobileToolsLite:
|
|
|
1834
2326
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
1835
2327
|
elements = self.client.xml_parser.parse(xml_string)
|
|
1836
2328
|
|
|
2329
|
+
# 功能控件类型(需要保留)
|
|
2330
|
+
FUNCTIONAL_WIDGETS = {
|
|
2331
|
+
'TextView', 'Text', 'Label', # 文本类
|
|
2332
|
+
'ImageView', 'Image', 'ImageButton', # 图片类
|
|
2333
|
+
'Button', 'CheckBox', 'RadioButton', 'Switch', # 交互类
|
|
2334
|
+
'SeekBar', 'ProgressBar', 'RatingBar', # 滑动/进度类
|
|
2335
|
+
'EditText', 'TextInput', # 输入类
|
|
2336
|
+
'VideoView', 'WebView', # 特殊功能类
|
|
2337
|
+
'RecyclerView', 'ListView', 'GridView', # 列表类
|
|
2338
|
+
'ScrollView', 'NestedScrollView', # 滚动容器(有实际功能)
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
# 容器控件类型(需要过滤,除非有业务ID)
|
|
2342
|
+
CONTAINER_WIDGETS = {
|
|
2343
|
+
'FrameLayout', 'LinearLayout', 'RelativeLayout',
|
|
2344
|
+
'ViewGroup', 'ConstraintLayout', 'CoordinatorLayout',
|
|
2345
|
+
'CardView', 'View', # 基础View也可能只是容器
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2348
|
+
# 装饰类控件关键词(resource_id中包含这些关键词的通常可以过滤)
|
|
2349
|
+
# 支持匹配如 qylt_item_short_video_shadow_one 这样的命名
|
|
2350
|
+
DECORATIVE_KEYWORDS = {
|
|
2351
|
+
'shadow', 'divider', 'separator', 'line', 'border',
|
|
2352
|
+
'background', 'bg_', '_bg', 'decorative', 'decoration',
|
|
2353
|
+
'_shadow', 'shadow_', '_divider', 'divider_', '_line', 'line_'
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
# Token 优化:构建精简元素(只返回非空字段)
|
|
2357
|
+
def build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name):
|
|
2358
|
+
"""只返回有值的字段,节省 token"""
|
|
2359
|
+
item = {}
|
|
2360
|
+
if resource_id:
|
|
2361
|
+
# 精简 resource_id,只保留最后一段
|
|
2362
|
+
item['id'] = resource_id.split('/')[-1] if '/' in resource_id else resource_id
|
|
2363
|
+
if text:
|
|
2364
|
+
item['text'] = text
|
|
2365
|
+
if content_desc:
|
|
2366
|
+
item['desc'] = content_desc
|
|
2367
|
+
if bounds:
|
|
2368
|
+
item['bounds'] = bounds
|
|
2369
|
+
if likely_click:
|
|
2370
|
+
item['click'] = True # 启发式判断可点击
|
|
2371
|
+
# class 精简:只保留关键类型
|
|
2372
|
+
if class_name in ('EditText', 'TextInput', 'Button', 'ImageButton', 'CheckBox', 'Switch'):
|
|
2373
|
+
item['type'] = class_name
|
|
2374
|
+
return item
|
|
2375
|
+
|
|
1837
2376
|
result = []
|
|
1838
2377
|
for elem in elements:
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
2378
|
+
# 获取元素属性
|
|
2379
|
+
class_name = elem.get('class_name', '')
|
|
2380
|
+
resource_id = elem.get('resource_id', '').strip()
|
|
2381
|
+
text = elem.get('text', '').strip()
|
|
2382
|
+
content_desc = elem.get('content_desc', '').strip()
|
|
2383
|
+
bounds = elem.get('bounds', '')
|
|
2384
|
+
clickable = elem.get('clickable', False)
|
|
2385
|
+
focusable = elem.get('focusable', False)
|
|
2386
|
+
scrollable = elem.get('scrollable', False)
|
|
2387
|
+
enabled = elem.get('enabled', True)
|
|
2388
|
+
|
|
2389
|
+
# 1. 过滤 bounds="[0,0][0,0]" 的视觉隐藏元素
|
|
2390
|
+
if bounds == '[0,0][0,0]':
|
|
2391
|
+
continue
|
|
2392
|
+
|
|
2393
|
+
# 2. 检查是否是功能控件(直接保留)
|
|
2394
|
+
if class_name in FUNCTIONAL_WIDGETS:
|
|
2395
|
+
# 使用启发式判断可点击性(替代不准确的 clickable 属性)
|
|
2396
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2397
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2398
|
+
if item:
|
|
2399
|
+
result.append(item)
|
|
2400
|
+
continue
|
|
2401
|
+
|
|
2402
|
+
# 3. 检查是否是容器控件
|
|
2403
|
+
if class_name in CONTAINER_WIDGETS:
|
|
2404
|
+
# 容器控件需要检查是否有业务相关的ID
|
|
2405
|
+
has_business_id = self._has_business_id(resource_id)
|
|
2406
|
+
if not has_business_id:
|
|
2407
|
+
# 无业务ID的容器控件,检查是否有其他有意义属性
|
|
2408
|
+
if not (clickable or focusable or scrollable or text or content_desc):
|
|
2409
|
+
# 所有属性都是默认值,过滤掉
|
|
2410
|
+
continue
|
|
2411
|
+
# 有业务ID或其他有意义属性,保留
|
|
2412
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2413
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2414
|
+
if item:
|
|
2415
|
+
result.append(item)
|
|
2416
|
+
continue
|
|
2417
|
+
|
|
2418
|
+
# 4. 检查是否是装饰类控件
|
|
2419
|
+
if resource_id:
|
|
2420
|
+
resource_id_lower = resource_id.lower()
|
|
2421
|
+
if any(keyword in resource_id_lower for keyword in DECORATIVE_KEYWORDS):
|
|
2422
|
+
# 是装饰类控件,且没有交互属性,过滤掉
|
|
2423
|
+
if not (clickable or focusable or text or content_desc):
|
|
2424
|
+
continue
|
|
2425
|
+
|
|
2426
|
+
# 5. 检查是否所有属性均为默认值
|
|
2427
|
+
if not (text or content_desc or resource_id or clickable or focusable or scrollable):
|
|
2428
|
+
# 所有属性都是默认值,过滤掉
|
|
2429
|
+
continue
|
|
2430
|
+
|
|
2431
|
+
# 6. 其他情况:有意义的元素保留
|
|
2432
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2433
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2434
|
+
if item:
|
|
2435
|
+
result.append(item)
|
|
2436
|
+
|
|
2437
|
+
# Token 优化:可选限制返回元素数量(默认不限制,确保准确度)
|
|
2438
|
+
if TOKEN_OPTIMIZATION and MAX_ELEMENTS > 0 and len(result) > MAX_ELEMENTS:
|
|
2439
|
+
# 仅在用户明确设置 MAX_ELEMENTS_RETURN 时才截断
|
|
2440
|
+
truncated = result[:MAX_ELEMENTS]
|
|
2441
|
+
truncated.append({
|
|
2442
|
+
'_truncated': True,
|
|
2443
|
+
'_total': len(result),
|
|
2444
|
+
'_shown': MAX_ELEMENTS
|
|
2445
|
+
})
|
|
2446
|
+
return truncated
|
|
2447
|
+
|
|
1847
2448
|
return result
|
|
1848
2449
|
except Exception as e:
|
|
1849
2450
|
return [{"error": f"获取元素失败: {e}"}]
|
|
1850
2451
|
|
|
2452
|
+
def _get_page_texts(self, max_count: int = 15) -> List[str]:
|
|
2453
|
+
"""获取页面关键文本列表(用于点击后快速确认页面变化)
|
|
2454
|
+
|
|
2455
|
+
Args:
|
|
2456
|
+
max_count: 最多返回的文本数量
|
|
2457
|
+
|
|
2458
|
+
Returns:
|
|
2459
|
+
页面上的关键文本列表(去重)
|
|
2460
|
+
"""
|
|
2461
|
+
try:
|
|
2462
|
+
if self._is_ios():
|
|
2463
|
+
ios_client = self._get_ios_client()
|
|
2464
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
2465
|
+
# iOS: 获取所有 StaticText 的文本
|
|
2466
|
+
elements = ios_client.wda(type='XCUIElementTypeStaticText').find_elements()
|
|
2467
|
+
texts = set()
|
|
2468
|
+
for elem in elements[:50]: # 限制扫描数量
|
|
2469
|
+
try:
|
|
2470
|
+
name = elem.name or elem.label
|
|
2471
|
+
if name and len(name) > 1 and len(name) < 50:
|
|
2472
|
+
texts.add(name)
|
|
2473
|
+
except:
|
|
2474
|
+
pass
|
|
2475
|
+
return list(texts)[:max_count]
|
|
2476
|
+
return []
|
|
2477
|
+
else:
|
|
2478
|
+
# Android: 快速扫描 XML 获取文本
|
|
2479
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=True)
|
|
2480
|
+
import xml.etree.ElementTree as ET
|
|
2481
|
+
root = ET.fromstring(xml_string)
|
|
2482
|
+
|
|
2483
|
+
texts = set()
|
|
2484
|
+
for elem in root.iter():
|
|
2485
|
+
text = elem.get('text', '').strip()
|
|
2486
|
+
desc = elem.get('content-desc', '').strip()
|
|
2487
|
+
# 只收集有意义的文本(长度2-30,非纯数字)
|
|
2488
|
+
for t in [text, desc]:
|
|
2489
|
+
if t and 2 <= len(t) <= 30 and not t.isdigit():
|
|
2490
|
+
texts.add(t)
|
|
2491
|
+
if len(texts) >= max_count * 2: # 收集足够后停止
|
|
2492
|
+
break
|
|
2493
|
+
|
|
2494
|
+
return list(texts)[:max_count]
|
|
2495
|
+
except Exception:
|
|
2496
|
+
return []
|
|
2497
|
+
|
|
2498
|
+
def _has_business_id(self, resource_id: str) -> bool:
|
|
2499
|
+
"""
|
|
2500
|
+
判断resource_id是否是业务相关的ID
|
|
2501
|
+
|
|
2502
|
+
业务相关的ID通常包含:
|
|
2503
|
+
- 有意义的命名(不是自动生成的)
|
|
2504
|
+
- 不包含常见的自动生成模式
|
|
2505
|
+
"""
|
|
2506
|
+
if not resource_id:
|
|
2507
|
+
return False
|
|
2508
|
+
|
|
2509
|
+
# 自动生成的ID模式(通常可以忽略)
|
|
2510
|
+
auto_generated_patterns = [
|
|
2511
|
+
r'^android:id/', # 系统ID
|
|
2512
|
+
r':id/\d+', # 数字ID
|
|
2513
|
+
r':id/view_\d+', # view_数字
|
|
2514
|
+
r':id/item_\d+', # item_数字
|
|
2515
|
+
]
|
|
2516
|
+
|
|
2517
|
+
for pattern in auto_generated_patterns:
|
|
2518
|
+
if re.search(pattern, resource_id):
|
|
2519
|
+
return False
|
|
2520
|
+
|
|
2521
|
+
# 如果resource_id有实际内容且不是自动生成的,认为是业务ID
|
|
2522
|
+
# 排除一些常见的系统ID
|
|
2523
|
+
system_ids = ['android:id/content', 'android:id/statusBarBackground']
|
|
2524
|
+
if resource_id in system_ids:
|
|
2525
|
+
return False
|
|
2526
|
+
|
|
2527
|
+
return True
|
|
2528
|
+
|
|
2529
|
+
def _is_likely_clickable(self, class_name: str, resource_id: str, text: str,
|
|
2530
|
+
content_desc: str, clickable: bool, bounds: str) -> bool:
|
|
2531
|
+
"""
|
|
2532
|
+
启发式判断元素是否可能可点击
|
|
2533
|
+
|
|
2534
|
+
Android 的 clickable 属性经常不准确,因为:
|
|
2535
|
+
1. 点击事件可能设置在父容器上
|
|
2536
|
+
2. 使用 onTouchListener 而不是 onClick
|
|
2537
|
+
3. RecyclerView item 通过 ItemClickListener 处理
|
|
2538
|
+
|
|
2539
|
+
此方法通过多种规则推断元素的真实可点击性
|
|
2540
|
+
"""
|
|
2541
|
+
# 规则1:clickable=true 肯定可点击
|
|
2542
|
+
if clickable:
|
|
2543
|
+
return True
|
|
2544
|
+
|
|
2545
|
+
# 规则2:特定类型的控件通常可点击
|
|
2546
|
+
TYPICALLY_CLICKABLE = {
|
|
2547
|
+
'Button', 'ImageButton', 'CheckBox', 'RadioButton', 'Switch',
|
|
2548
|
+
'ToggleButton', 'FloatingActionButton', 'Chip', 'TabView',
|
|
2549
|
+
'EditText', 'TextInput', # 输入框可点击获取焦点
|
|
2550
|
+
}
|
|
2551
|
+
if class_name in TYPICALLY_CLICKABLE:
|
|
2552
|
+
return True
|
|
2553
|
+
|
|
2554
|
+
# 规则3:resource_id 包含可点击关键词
|
|
2555
|
+
if resource_id:
|
|
2556
|
+
id_lower = resource_id.lower()
|
|
2557
|
+
CLICK_KEYWORDS = [
|
|
2558
|
+
'btn', 'button', 'click', 'tap', 'submit', 'confirm',
|
|
2559
|
+
'cancel', 'close', 'back', 'next', 'prev', 'more',
|
|
2560
|
+
'action', 'link', 'menu', 'tab', 'item', 'cell',
|
|
2561
|
+
'card', 'avatar', 'icon', 'entry', 'option', 'arrow'
|
|
2562
|
+
]
|
|
2563
|
+
for kw in CLICK_KEYWORDS:
|
|
2564
|
+
if kw in id_lower:
|
|
2565
|
+
return True
|
|
2566
|
+
|
|
2567
|
+
# 规则4:content_desc 包含可点击暗示
|
|
2568
|
+
if content_desc:
|
|
2569
|
+
desc_lower = content_desc.lower()
|
|
2570
|
+
CLICK_HINTS = ['点击', '按钮', '关闭', '返回', '更多', 'click', 'tap', 'button', 'close']
|
|
2571
|
+
for hint in CLICK_HINTS:
|
|
2572
|
+
if hint in desc_lower:
|
|
2573
|
+
return True
|
|
2574
|
+
|
|
2575
|
+
# 规则5:有 resource_id 或 content_desc 的小图标可能可点击
|
|
2576
|
+
# (纯 ImageView 不加判断,误判率太高)
|
|
2577
|
+
if class_name in ('ImageView', 'Image') and (resource_id or content_desc) and bounds:
|
|
2578
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
2579
|
+
if match:
|
|
2580
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2581
|
+
w, h = x2 - x1, y2 - y1
|
|
2582
|
+
# 小图标(20-100px)更可能是按钮
|
|
2583
|
+
if 20 <= w <= 100 and 20 <= h <= 100:
|
|
2584
|
+
return True
|
|
2585
|
+
|
|
2586
|
+
# 规则6:移除(TextView 误判率太高,只依赖上面的规则)
|
|
2587
|
+
# 如果有 clickable=true 或 ID/desc 中有关键词,前面的规则已经覆盖
|
|
2588
|
+
|
|
2589
|
+
return False
|
|
2590
|
+
|
|
1851
2591
|
def find_close_button(self) -> Dict:
|
|
1852
2592
|
"""智能查找关闭按钮(不点击,只返回位置)
|
|
1853
2593
|
|
|
@@ -1861,7 +2601,7 @@ class BasicMobileToolsLite:
|
|
|
1861
2601
|
import re
|
|
1862
2602
|
|
|
1863
2603
|
if self._is_ios():
|
|
1864
|
-
return {"success": False, "
|
|
2604
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
1865
2605
|
|
|
1866
2606
|
# 获取屏幕尺寸
|
|
1867
2607
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
@@ -1872,6 +2612,14 @@ class BasicMobileToolsLite:
|
|
|
1872
2612
|
import xml.etree.ElementTree as ET
|
|
1873
2613
|
root = ET.fromstring(xml_string)
|
|
1874
2614
|
|
|
2615
|
+
# 🔴 先检测是否有弹窗,避免误识别普通页面的按钮
|
|
2616
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2617
|
+
root, screen_width, screen_height
|
|
2618
|
+
)
|
|
2619
|
+
|
|
2620
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
2621
|
+
return {"success": True, "popup": False}
|
|
2622
|
+
|
|
1875
2623
|
# 关闭按钮特征
|
|
1876
2624
|
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
|
|
1877
2625
|
candidates = []
|
|
@@ -1973,27 +2721,16 @@ class BasicMobileToolsLite:
|
|
|
1973
2721
|
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
1974
2722
|
best = candidates[0]
|
|
1975
2723
|
|
|
2724
|
+
# Token 优化:只返回最必要的信息
|
|
1976
2725
|
return {
|
|
1977
2726
|
"success": True,
|
|
1978
|
-
"
|
|
1979
|
-
"
|
|
1980
|
-
|
|
1981
|
-
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
1982
|
-
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
1983
|
-
"bounds": best['bounds'],
|
|
1984
|
-
"size": best['size'],
|
|
1985
|
-
"score": best['score']
|
|
1986
|
-
},
|
|
1987
|
-
"click_command": f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})",
|
|
1988
|
-
"other_candidates": [
|
|
1989
|
-
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
1990
|
-
for c in candidates[1:4]
|
|
1991
|
-
] if len(candidates) > 1 else [],
|
|
1992
|
-
"screen_size": {"width": screen_width, "height": screen_height}
|
|
2727
|
+
"popup": True,
|
|
2728
|
+
"close": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2729
|
+
"cmd": f"click_by_percent({best['x_percent']},{best['y_percent']})"
|
|
1993
2730
|
}
|
|
1994
2731
|
|
|
1995
2732
|
except Exception as e:
|
|
1996
|
-
return {"success": False, "
|
|
2733
|
+
return {"success": False, "msg": str(e)}
|
|
1997
2734
|
|
|
1998
2735
|
def close_popup(self) -> Dict:
|
|
1999
2736
|
"""智能关闭弹窗(改进版)
|
|
@@ -2016,7 +2753,7 @@ class BasicMobileToolsLite:
|
|
|
2016
2753
|
|
|
2017
2754
|
# 获取屏幕尺寸
|
|
2018
2755
|
if self._is_ios():
|
|
2019
|
-
return {"success": False, "
|
|
2756
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2020
2757
|
|
|
2021
2758
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2022
2759
|
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
@@ -2036,53 +2773,18 @@ class BasicMobileToolsLite:
|
|
|
2036
2773
|
root = ET.fromstring(xml_string)
|
|
2037
2774
|
all_elements = list(root.iter())
|
|
2038
2775
|
|
|
2039
|
-
# =====
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
if not bounds_str:
|
|
2047
|
-
continue
|
|
2048
|
-
|
|
2049
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2050
|
-
if not match:
|
|
2051
|
-
continue
|
|
2052
|
-
|
|
2053
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2054
|
-
width = x2 - x1
|
|
2055
|
-
height = y2 - y1
|
|
2056
|
-
area = width * height
|
|
2057
|
-
screen_area = screen_width * screen_height
|
|
2058
|
-
|
|
2059
|
-
# 弹窗容器特征:
|
|
2060
|
-
# 1. 面积在屏幕的 10%-90% 之间(非全屏)
|
|
2061
|
-
# 2. 宽度或高度不等于屏幕尺寸
|
|
2062
|
-
# 3. 是容器类型(Layout/View/Dialog)
|
|
2063
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Container'])
|
|
2064
|
-
area_ratio = area / screen_area
|
|
2065
|
-
is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
|
|
2066
|
-
is_reasonable_size = 0.08 < area_ratio < 0.9
|
|
2067
|
-
|
|
2068
|
-
# 排除状态栏区域(y1 通常很小)
|
|
2069
|
-
is_below_statusbar = y1 > 50
|
|
2070
|
-
|
|
2071
|
-
if is_container and is_not_fullscreen and is_reasonable_size and is_below_statusbar:
|
|
2072
|
-
popup_containers.append({
|
|
2073
|
-
'bounds': (x1, y1, x2, y2),
|
|
2074
|
-
'bounds_str': bounds_str,
|
|
2075
|
-
'area': area,
|
|
2076
|
-
'area_ratio': area_ratio,
|
|
2077
|
-
'idx': idx, # 元素在 XML 中的顺序(越后越上层)
|
|
2078
|
-
'class': class_name
|
|
2079
|
-
})
|
|
2776
|
+
# ===== 第一步:使用严格的置信度检测弹窗区域 =====
|
|
2777
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2778
|
+
root, screen_width, screen_height
|
|
2779
|
+
)
|
|
2780
|
+
|
|
2781
|
+
# 如果置信度不够高,记录但继续尝试查找关闭按钮
|
|
2782
|
+
popup_detected = popup_bounds is not None and popup_confidence >= 0.6
|
|
2080
2783
|
|
|
2081
|
-
#
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
popup_bounds = popup_containers[0]['bounds']
|
|
2784
|
+
# 🔴 关键检查:如果没有检测到弹窗区域,直接返回"无弹窗"
|
|
2785
|
+
# 避免误点击普通页面上的"关闭"、"取消"等按钮
|
|
2786
|
+
if not popup_detected:
|
|
2787
|
+
return {"success": True, "popup": False}
|
|
2086
2788
|
|
|
2087
2789
|
# ===== 第二步:在弹窗范围内查找关闭按钮 =====
|
|
2088
2790
|
for idx, elem in enumerate(all_elements):
|
|
@@ -2214,73 +2916,16 @@ class BasicMobileToolsLite:
|
|
|
2214
2916
|
'content_desc': content_desc,
|
|
2215
2917
|
'x_percent': round(rel_x * 100, 1),
|
|
2216
2918
|
'y_percent': round(rel_y * 100, 1),
|
|
2217
|
-
'in_popup':
|
|
2919
|
+
'in_popup': popup_detected
|
|
2218
2920
|
})
|
|
2219
2921
|
|
|
2220
2922
|
except ET.ParseError:
|
|
2221
2923
|
pass
|
|
2222
2924
|
|
|
2223
2925
|
if not close_candidates:
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
popup_width = px2 - px1
|
|
2228
|
-
popup_height = py2 - py1
|
|
2229
|
-
|
|
2230
|
-
# 【优化】X按钮有三种常见位置:
|
|
2231
|
-
# 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
|
|
2232
|
-
# 2. 弹窗边界上方(浮动X按钮)
|
|
2233
|
-
# 3. 弹窗正下方(底部关闭按钮)
|
|
2234
|
-
offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
|
|
2235
|
-
offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
|
|
2236
|
-
offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
|
|
2237
|
-
|
|
2238
|
-
try_positions = [
|
|
2239
|
-
# 【最高优先级】弹窗内紧贴顶部边界
|
|
2240
|
-
(px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
|
|
2241
|
-
# 弹窗边界上方(浮动X按钮)
|
|
2242
|
-
(px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
|
|
2243
|
-
# 弹窗正下方中间(底部关闭按钮)
|
|
2244
|
-
((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
|
|
2245
|
-
# 弹窗正上方中间
|
|
2246
|
-
((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
|
|
2247
|
-
]
|
|
2248
|
-
|
|
2249
|
-
for try_x, try_y, position_name in try_positions:
|
|
2250
|
-
if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
|
|
2251
|
-
self.client.u2.click(try_x, try_y)
|
|
2252
|
-
time.sleep(0.3)
|
|
2253
|
-
|
|
2254
|
-
# 尝试后截图,让 AI 判断是否成功
|
|
2255
|
-
screenshot_result = self.take_screenshot("尝试关闭后")
|
|
2256
|
-
return {
|
|
2257
|
-
"success": True,
|
|
2258
|
-
"message": f"✅ 已尝试点击常见关闭按钮位置",
|
|
2259
|
-
"tried_positions": [p[2] for p in try_positions],
|
|
2260
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2261
|
-
"tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
|
|
2262
|
-
}
|
|
2263
|
-
|
|
2264
|
-
# 没有检测到弹窗区域,截图让 AI 分析
|
|
2265
|
-
screenshot_result = self.take_screenshot(description="页面截图", compress=True)
|
|
2266
|
-
|
|
2267
|
-
return {
|
|
2268
|
-
"success": False,
|
|
2269
|
-
"message": "❌ 未检测到弹窗区域,已截图供 AI 分析",
|
|
2270
|
-
"action_required": "请查看截图找到关闭按钮,调用 mobile_click_at_coords 点击",
|
|
2271
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2272
|
-
"screen_size": {"width": screen_width, "height": screen_height},
|
|
2273
|
-
"image_size": {
|
|
2274
|
-
"width": screenshot_result.get("image_width", screen_width),
|
|
2275
|
-
"height": screenshot_result.get("image_height", screen_height)
|
|
2276
|
-
},
|
|
2277
|
-
"original_size": {
|
|
2278
|
-
"width": screenshot_result.get("original_img_width", screen_width),
|
|
2279
|
-
"height": screenshot_result.get("original_img_height", screen_height)
|
|
2280
|
-
},
|
|
2281
|
-
"search_areas": ["弹窗右上角", "弹窗正上方", "弹窗下方中间", "屏幕右上角"],
|
|
2282
|
-
"time_warning": "⚠️ 截图分析期间弹窗可能自动消失。如果是定时弹窗,建议等待其自动消失。"
|
|
2283
|
-
}
|
|
2926
|
+
if popup_detected and popup_bounds:
|
|
2927
|
+
return {"success": False, "fallback": "vision", "popup": True}
|
|
2928
|
+
return {"success": True, "popup": False}
|
|
2284
2929
|
|
|
2285
2930
|
# 按得分排序,取最可能的
|
|
2286
2931
|
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
@@ -2290,49 +2935,30 @@ class BasicMobileToolsLite:
|
|
|
2290
2935
|
self.client.u2.click(best['center_x'], best['center_y'])
|
|
2291
2936
|
time.sleep(0.5)
|
|
2292
2937
|
|
|
2293
|
-
#
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
# 记录操作(使用百分比,跨设备兼容)
|
|
2297
|
-
self._record_operation(
|
|
2298
|
-
'click',
|
|
2299
|
-
x=best['center_x'],
|
|
2300
|
-
y=best['center_y'],
|
|
2301
|
-
x_percent=best['x_percent'],
|
|
2302
|
-
y_percent=best['y_percent'],
|
|
2303
|
-
screen_width=screen_width,
|
|
2304
|
-
screen_height=screen_height,
|
|
2305
|
-
ref=f"close_popup_{best['position']}"
|
|
2306
|
-
)
|
|
2938
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
|
|
2939
|
+
app_check = self._check_app_switched()
|
|
2940
|
+
return_result = None
|
|
2307
2941
|
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
"
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
"type": c['match_type'],
|
|
2326
|
-
"coords": (c['center_x'], c['center_y']),
|
|
2327
|
-
"percent": (c['x_percent'], c['y_percent'])
|
|
2328
|
-
}
|
|
2329
|
-
for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
|
|
2330
|
-
],
|
|
2331
|
-
"tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置;如果误点跳转了,请按返回键"
|
|
2332
|
-
}
|
|
2942
|
+
if app_check['switched']:
|
|
2943
|
+
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
2944
|
+
return_result = self._return_to_target_app()
|
|
2945
|
+
|
|
2946
|
+
# 记录操作
|
|
2947
|
+
self._record_click('percent', f"{best['x_percent']}%,{best['y_percent']}%",
|
|
2948
|
+
best['x_percent'], best['y_percent'],
|
|
2949
|
+
element_desc=f"关闭按钮({best['position']})")
|
|
2950
|
+
|
|
2951
|
+
# Token 优化:精简返回值
|
|
2952
|
+
result = {"success": True, "clicked": True}
|
|
2953
|
+
if app_check['switched']:
|
|
2954
|
+
result["switched"] = True
|
|
2955
|
+
if return_result:
|
|
2956
|
+
result["returned"] = return_result['success']
|
|
2957
|
+
|
|
2958
|
+
return result
|
|
2333
2959
|
|
|
2334
2960
|
except Exception as e:
|
|
2335
|
-
return {"success": False, "
|
|
2961
|
+
return {"success": False, "msg": str(e)}
|
|
2336
2962
|
|
|
2337
2963
|
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
2338
2964
|
"""根据相对坐标获取位置名称"""
|
|
@@ -2375,6 +3001,308 @@ class BasicMobileToolsLite:
|
|
|
2375
3001
|
return 0.8
|
|
2376
3002
|
else: # 中间区域
|
|
2377
3003
|
return 0.5
|
|
3004
|
+
|
|
3005
|
+
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) -> tuple:
|
|
3006
|
+
"""严格的弹窗检测 - 使用置信度评分,避免误识别普通页面
|
|
3007
|
+
|
|
3008
|
+
真正的弹窗特征:
|
|
3009
|
+
1. class 名称包含 Dialog/Popup/Alert/Modal/BottomSheet(强特征)
|
|
3010
|
+
2. resource-id 包含 dialog/popup/alert/modal(强特征)
|
|
3011
|
+
3. 有遮罩层(大面积半透明 View 在弹窗之前)
|
|
3012
|
+
4. 居中显示且非全屏
|
|
3013
|
+
5. XML 层级靠后且包含可交互元素
|
|
3014
|
+
|
|
3015
|
+
Returns:
|
|
3016
|
+
(popup_bounds, confidence) 或 (None, 0)
|
|
3017
|
+
confidence >= 0.6 才认为是弹窗
|
|
3018
|
+
"""
|
|
3019
|
+
import re
|
|
3020
|
+
|
|
3021
|
+
screen_area = screen_width * screen_height
|
|
3022
|
+
|
|
3023
|
+
# 收集所有元素信息
|
|
3024
|
+
all_elements = []
|
|
3025
|
+
for idx, elem in enumerate(root.iter()):
|
|
3026
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
3027
|
+
if not bounds_str:
|
|
3028
|
+
continue
|
|
3029
|
+
|
|
3030
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
3031
|
+
if not match:
|
|
3032
|
+
continue
|
|
3033
|
+
|
|
3034
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
3035
|
+
width = x2 - x1
|
|
3036
|
+
height = y2 - y1
|
|
3037
|
+
area = width * height
|
|
3038
|
+
|
|
3039
|
+
class_name = elem.attrib.get('class', '')
|
|
3040
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
3041
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
3042
|
+
|
|
3043
|
+
all_elements.append({
|
|
3044
|
+
'idx': idx,
|
|
3045
|
+
'bounds': (x1, y1, x2, y2),
|
|
3046
|
+
'width': width,
|
|
3047
|
+
'height': height,
|
|
3048
|
+
'area': area,
|
|
3049
|
+
'area_ratio': area / screen_area if screen_area > 0 else 0,
|
|
3050
|
+
'class': class_name,
|
|
3051
|
+
'resource_id': resource_id,
|
|
3052
|
+
'clickable': clickable,
|
|
3053
|
+
'center_x': (x1 + x2) // 2,
|
|
3054
|
+
'center_y': (y1 + y2) // 2,
|
|
3055
|
+
})
|
|
3056
|
+
|
|
3057
|
+
if not all_elements:
|
|
3058
|
+
return None, 0
|
|
3059
|
+
|
|
3060
|
+
# 弹窗检测关键词
|
|
3061
|
+
dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
|
|
3062
|
+
dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
|
|
3063
|
+
|
|
3064
|
+
popup_candidates = []
|
|
3065
|
+
has_mask_layer = False
|
|
3066
|
+
mask_idx = -1
|
|
3067
|
+
|
|
3068
|
+
for elem in all_elements:
|
|
3069
|
+
x1, y1, x2, y2 = elem['bounds']
|
|
3070
|
+
class_name = elem['class']
|
|
3071
|
+
resource_id = elem['resource_id']
|
|
3072
|
+
area_ratio = elem['area_ratio']
|
|
3073
|
+
|
|
3074
|
+
# 检测遮罩层(大面积、几乎全屏、通常是 FrameLayout/View)
|
|
3075
|
+
if area_ratio > 0.85 and elem['width'] >= screen_width * 0.95:
|
|
3076
|
+
# 可能是遮罩层,记录位置
|
|
3077
|
+
if 'FrameLayout' in class_name or 'View' in class_name:
|
|
3078
|
+
has_mask_layer = True
|
|
3079
|
+
mask_idx = elem['idx']
|
|
3080
|
+
|
|
3081
|
+
# 跳过全屏元素
|
|
3082
|
+
if area_ratio > 0.9:
|
|
3083
|
+
continue
|
|
3084
|
+
|
|
3085
|
+
# 跳过太小的元素
|
|
3086
|
+
if area_ratio < 0.05:
|
|
3087
|
+
continue
|
|
3088
|
+
|
|
3089
|
+
# 跳过状态栏区域
|
|
3090
|
+
if y1 < 50:
|
|
3091
|
+
continue
|
|
3092
|
+
|
|
3093
|
+
confidence = 0.0
|
|
3094
|
+
|
|
3095
|
+
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
3096
|
+
if any(kw in class_name for kw in dialog_class_keywords):
|
|
3097
|
+
confidence += 0.5
|
|
3098
|
+
|
|
3099
|
+
# 【强特征】resource-id 包含弹窗关键词 (+0.4)
|
|
3100
|
+
if any(kw in resource_id.lower() for kw in dialog_id_keywords):
|
|
3101
|
+
confidence += 0.4
|
|
3102
|
+
|
|
3103
|
+
# 【中等特征】居中显示 (+0.2)
|
|
3104
|
+
center_x = elem['center_x']
|
|
3105
|
+
center_y = elem['center_y']
|
|
3106
|
+
is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
|
|
3107
|
+
is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
|
|
3108
|
+
if is_centered_x and is_centered_y:
|
|
3109
|
+
confidence += 0.2
|
|
3110
|
+
elif is_centered_x:
|
|
3111
|
+
confidence += 0.1
|
|
3112
|
+
|
|
3113
|
+
# 【中等特征】非全屏但有一定大小 (+0.15)
|
|
3114
|
+
if 0.15 < area_ratio < 0.75:
|
|
3115
|
+
confidence += 0.15
|
|
3116
|
+
|
|
3117
|
+
# 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
|
|
3118
|
+
if elem['idx'] > len(all_elements) * 0.5:
|
|
3119
|
+
confidence += 0.1
|
|
3120
|
+
|
|
3121
|
+
# 【弱特征】有遮罩层且在遮罩层之后 (+0.15)
|
|
3122
|
+
if has_mask_layer and elem['idx'] > mask_idx:
|
|
3123
|
+
confidence += 0.15
|
|
3124
|
+
|
|
3125
|
+
# 只有达到阈值才加入候选
|
|
3126
|
+
if confidence >= 0.3:
|
|
3127
|
+
popup_candidates.append({
|
|
3128
|
+
'bounds': elem['bounds'],
|
|
3129
|
+
'confidence': confidence,
|
|
3130
|
+
'class': class_name,
|
|
3131
|
+
'resource_id': resource_id,
|
|
3132
|
+
'idx': elem['idx']
|
|
3133
|
+
})
|
|
3134
|
+
|
|
3135
|
+
if not popup_candidates:
|
|
3136
|
+
return None, 0
|
|
3137
|
+
|
|
3138
|
+
# 选择置信度最高的
|
|
3139
|
+
popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
|
|
3140
|
+
best = popup_candidates[0]
|
|
3141
|
+
|
|
3142
|
+
# 只有置信度 >= 0.6 才返回弹窗
|
|
3143
|
+
if best['confidence'] >= 0.6:
|
|
3144
|
+
return best['bounds'], best['confidence']
|
|
3145
|
+
|
|
3146
|
+
return None, best['confidence']
|
|
3147
|
+
|
|
3148
|
+
def start_toast_watch(self) -> Dict:
|
|
3149
|
+
"""开始监听 Toast(仅 Android)
|
|
3150
|
+
|
|
3151
|
+
⚠️ 必须在执行操作之前调用!
|
|
3152
|
+
|
|
3153
|
+
正确流程:
|
|
3154
|
+
1. 调用 mobile_start_toast_watch() 开始监听
|
|
3155
|
+
2. 执行操作(如点击提交按钮)
|
|
3156
|
+
3. 调用 mobile_get_toast() 获取 Toast 内容
|
|
3157
|
+
|
|
3158
|
+
Returns:
|
|
3159
|
+
监听状态
|
|
3160
|
+
"""
|
|
3161
|
+
if self._is_ios():
|
|
3162
|
+
return {
|
|
3163
|
+
"success": False,
|
|
3164
|
+
"message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
|
|
3165
|
+
}
|
|
3166
|
+
|
|
3167
|
+
try:
|
|
3168
|
+
# 清除缓存并开始监听
|
|
3169
|
+
self.client.u2.toast.reset()
|
|
3170
|
+
return {
|
|
3171
|
+
"success": True,
|
|
3172
|
+
"message": "✅ Toast 监听已开启,请立即执行操作,然后调用 mobile_get_toast 获取结果"
|
|
3173
|
+
}
|
|
3174
|
+
except Exception as e:
|
|
3175
|
+
return {
|
|
3176
|
+
"success": False,
|
|
3177
|
+
"message": f"❌ 开启 Toast 监听失败: {e}"
|
|
3178
|
+
}
|
|
3179
|
+
|
|
3180
|
+
def get_toast(self, timeout: float = 5.0, reset_first: bool = False) -> Dict:
|
|
3181
|
+
"""获取 Toast 消息(仅 Android)
|
|
3182
|
+
|
|
3183
|
+
Toast 是 Android 系统级的短暂提示消息,常用于显示操作结果。
|
|
3184
|
+
|
|
3185
|
+
⚠️ 推荐用法(两步走):
|
|
3186
|
+
1. 先调用 mobile_start_toast_watch() 开始监听
|
|
3187
|
+
2. 执行操作(如点击提交按钮)
|
|
3188
|
+
3. 调用 mobile_get_toast() 获取 Toast
|
|
3189
|
+
|
|
3190
|
+
或者设置 reset_first=True,会自动 reset 后等待(适合操作已自动触发的场景)
|
|
3191
|
+
|
|
3192
|
+
Args:
|
|
3193
|
+
timeout: 等待 Toast 出现的超时时间(秒),默认 5 秒
|
|
3194
|
+
reset_first: 是否先 reset(清除旧缓存),默认 False
|
|
3195
|
+
|
|
3196
|
+
Returns:
|
|
3197
|
+
包含 Toast 消息的字典
|
|
3198
|
+
"""
|
|
3199
|
+
if self._is_ios():
|
|
3200
|
+
return {
|
|
3201
|
+
"success": False,
|
|
3202
|
+
"message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
|
|
3203
|
+
}
|
|
3204
|
+
|
|
3205
|
+
try:
|
|
3206
|
+
if reset_first:
|
|
3207
|
+
# 清除旧缓存,适合等待即将出现的 Toast
|
|
3208
|
+
self.client.u2.toast.reset()
|
|
3209
|
+
|
|
3210
|
+
# 等待并获取 Toast 消息
|
|
3211
|
+
toast_message = self.client.u2.toast.get_message(
|
|
3212
|
+
wait_timeout=timeout,
|
|
3213
|
+
default=None
|
|
3214
|
+
)
|
|
3215
|
+
|
|
3216
|
+
if toast_message:
|
|
3217
|
+
return {
|
|
3218
|
+
"success": True,
|
|
3219
|
+
"toast_found": True,
|
|
3220
|
+
"message": toast_message,
|
|
3221
|
+
"tip": "Toast 消息获取成功"
|
|
3222
|
+
}
|
|
3223
|
+
else:
|
|
3224
|
+
return {
|
|
3225
|
+
"success": True,
|
|
3226
|
+
"toast_found": False,
|
|
3227
|
+
"message": None,
|
|
3228
|
+
"tip": f"在 {timeout} 秒内未检测到 Toast。提示:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具"
|
|
3229
|
+
}
|
|
3230
|
+
except Exception as e:
|
|
3231
|
+
return {
|
|
3232
|
+
"success": False,
|
|
3233
|
+
"message": f"❌ 获取 Toast 失败: {e}"
|
|
3234
|
+
}
|
|
3235
|
+
|
|
3236
|
+
def assert_toast(self, expected_text: str, timeout: float = 5.0, contains: bool = True) -> Dict:
|
|
3237
|
+
"""断言 Toast 消息(仅 Android)
|
|
3238
|
+
|
|
3239
|
+
等待 Toast 出现并验证内容是否符合预期。
|
|
3240
|
+
|
|
3241
|
+
⚠️ 推荐用法:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具
|
|
3242
|
+
|
|
3243
|
+
Args:
|
|
3244
|
+
expected_text: 期望的 Toast 文本
|
|
3245
|
+
timeout: 等待超时时间(秒)
|
|
3246
|
+
contains: True 表示包含匹配,False 表示精确匹配
|
|
3247
|
+
|
|
3248
|
+
Returns:
|
|
3249
|
+
断言结果
|
|
3250
|
+
"""
|
|
3251
|
+
if self._is_ios():
|
|
3252
|
+
return {
|
|
3253
|
+
"success": False,
|
|
3254
|
+
"passed": False,
|
|
3255
|
+
"message": "❌ iOS 不支持 Toast 检测"
|
|
3256
|
+
}
|
|
3257
|
+
|
|
3258
|
+
try:
|
|
3259
|
+
# 获取 Toast(不 reset,假设之前已经调用过 start_toast_watch)
|
|
3260
|
+
toast_message = self.client.u2.toast.get_message(
|
|
3261
|
+
wait_timeout=timeout,
|
|
3262
|
+
default=None
|
|
3263
|
+
)
|
|
3264
|
+
|
|
3265
|
+
if toast_message is None:
|
|
3266
|
+
return {
|
|
3267
|
+
"success": True,
|
|
3268
|
+
"passed": False,
|
|
3269
|
+
"expected": expected_text,
|
|
3270
|
+
"actual": None,
|
|
3271
|
+
"message": f"❌ 断言失败:未检测到 Toast 消息"
|
|
3272
|
+
}
|
|
3273
|
+
|
|
3274
|
+
# 匹配检查
|
|
3275
|
+
if contains:
|
|
3276
|
+
passed = expected_text in toast_message
|
|
3277
|
+
match_type = "包含"
|
|
3278
|
+
else:
|
|
3279
|
+
passed = expected_text == toast_message
|
|
3280
|
+
match_type = "精确"
|
|
3281
|
+
|
|
3282
|
+
if passed:
|
|
3283
|
+
return {
|
|
3284
|
+
"success": True,
|
|
3285
|
+
"passed": True,
|
|
3286
|
+
"expected": expected_text,
|
|
3287
|
+
"actual": toast_message,
|
|
3288
|
+
"match_type": match_type,
|
|
3289
|
+
"message": f"✅ Toast 断言通过:'{toast_message}'"
|
|
3290
|
+
}
|
|
3291
|
+
else:
|
|
3292
|
+
return {
|
|
3293
|
+
"success": True,
|
|
3294
|
+
"passed": False,
|
|
3295
|
+
"expected": expected_text,
|
|
3296
|
+
"actual": toast_message,
|
|
3297
|
+
"match_type": match_type,
|
|
3298
|
+
"message": f"❌ Toast 断言失败:期望 '{expected_text}',实际 '{toast_message}'"
|
|
3299
|
+
}
|
|
3300
|
+
except Exception as e:
|
|
3301
|
+
return {
|
|
3302
|
+
"success": False,
|
|
3303
|
+
"passed": False,
|
|
3304
|
+
"message": f"❌ Toast 断言异常: {e}"
|
|
3305
|
+
}
|
|
2378
3306
|
|
|
2379
3307
|
def assert_text(self, text: str) -> Dict:
|
|
2380
3308
|
"""检查页面是否包含文本(支持精确匹配和包含匹配)"""
|
|
@@ -2460,11 +3388,16 @@ class BasicMobileToolsLite:
|
|
|
2460
3388
|
f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
|
2461
3389
|
"",
|
|
2462
3390
|
"定位策略(按优先级):",
|
|
2463
|
-
"1.
|
|
2464
|
-
"2.
|
|
3391
|
+
"1. 文本定位 - 最稳定,跨设备兼容",
|
|
3392
|
+
"2. ID 定位 - 稳定,跨设备兼容",
|
|
2465
3393
|
"3. 百分比定位 - 跨分辨率兼容(坐标自动转换)",
|
|
3394
|
+
"",
|
|
3395
|
+
"运行方式:",
|
|
3396
|
+
" pytest {filename} -v # 使用 pytest 运行",
|
|
3397
|
+
" python {filename} # 直接运行",
|
|
2466
3398
|
f'"""',
|
|
2467
3399
|
"import time",
|
|
3400
|
+
"import pytest",
|
|
2468
3401
|
"import uiautomator2 as u2",
|
|
2469
3402
|
"",
|
|
2470
3403
|
f'PACKAGE_NAME = "{package_name}"',
|
|
@@ -2540,22 +3473,52 @@ class BasicMobileToolsLite:
|
|
|
2540
3473
|
" return True",
|
|
2541
3474
|
"",
|
|
2542
3475
|
"",
|
|
2543
|
-
"def
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
3476
|
+
"def swipe_direction(d, direction):",
|
|
3477
|
+
' """',
|
|
3478
|
+
' 通用滑动方法(兼容所有 uiautomator2 版本)',
|
|
3479
|
+
' ',
|
|
3480
|
+
' Args:',
|
|
3481
|
+
' d: uiautomator2 设备对象',
|
|
3482
|
+
' direction: 滑动方向 (up/down/left/right)',
|
|
3483
|
+
' """',
|
|
3484
|
+
" info = d.info",
|
|
3485
|
+
" width = info.get('displayWidth', 0)",
|
|
3486
|
+
" height = info.get('displayHeight', 0)",
|
|
3487
|
+
" cx, cy = width // 2, height // 2",
|
|
2551
3488
|
" ",
|
|
2552
|
-
"
|
|
3489
|
+
" if direction == 'up':",
|
|
3490
|
+
" d.swipe(cx, int(height * 0.8), cx, int(height * 0.3))",
|
|
3491
|
+
" elif direction == 'down':",
|
|
3492
|
+
" d.swipe(cx, int(height * 0.3), cx, int(height * 0.8))",
|
|
3493
|
+
" elif direction == 'left':",
|
|
3494
|
+
" d.swipe(int(width * 0.8), cy, int(width * 0.2), cy)",
|
|
3495
|
+
" elif direction == 'right':",
|
|
3496
|
+
" d.swipe(int(width * 0.2), cy, int(width * 0.8), cy)",
|
|
3497
|
+
" return True",
|
|
3498
|
+
"",
|
|
3499
|
+
"",
|
|
3500
|
+
"# ========== pytest fixture ==========",
|
|
3501
|
+
"@pytest.fixture(scope='function')",
|
|
3502
|
+
"def device():",
|
|
3503
|
+
' """pytest fixture: 连接设备并启动应用"""',
|
|
3504
|
+
" d = u2.connect()",
|
|
3505
|
+
" d.implicitly_wait(10)",
|
|
3506
|
+
" d.app_start(PACKAGE_NAME)",
|
|
3507
|
+
" time.sleep(LAUNCH_WAIT)",
|
|
2553
3508
|
" if CLOSE_AD_ON_LAUNCH:",
|
|
2554
3509
|
" close_ad_if_exists(d)",
|
|
3510
|
+
" yield d",
|
|
3511
|
+
" # 测试结束后可选择关闭应用",
|
|
3512
|
+
" # d.app_stop(PACKAGE_NAME)",
|
|
3513
|
+
"",
|
|
3514
|
+
"",
|
|
3515
|
+
f"def test_{safe_name}(device):",
|
|
3516
|
+
' """测试用例主函数"""',
|
|
3517
|
+
" d = device",
|
|
2555
3518
|
" ",
|
|
2556
3519
|
]
|
|
2557
3520
|
|
|
2558
|
-
#
|
|
3521
|
+
# 生成操作代码(使用标准记录格式,逻辑更简洁)
|
|
2559
3522
|
step_num = 0
|
|
2560
3523
|
for op in self.operation_history:
|
|
2561
3524
|
action = op.get('action')
|
|
@@ -2567,130 +3530,122 @@ class BasicMobileToolsLite:
|
|
|
2567
3530
|
step_num += 1
|
|
2568
3531
|
|
|
2569
3532
|
if action == 'click':
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
script_lines.append(f"
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
script_lines.append(f"
|
|
2600
|
-
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
2601
|
-
elif has_coords:
|
|
2602
|
-
# 4️⃣ 坐标兜底(不推荐,仅用于无法获取百分比的情况)
|
|
2603
|
-
desc = f" ({element})" if element else ""
|
|
2604
|
-
script_lines.append(f" # 步骤{step_num}: 点击坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
2605
|
-
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
3533
|
+
# 新格式:使用 locator_type 和 locator_value
|
|
3534
|
+
locator_type = op.get('locator_type', '')
|
|
3535
|
+
locator_value = op.get('locator_value', '')
|
|
3536
|
+
locator_attr = op.get('locator_attr', 'text')
|
|
3537
|
+
element_desc = op.get('element_desc', '')
|
|
3538
|
+
x_pct = op.get('x_percent', 0)
|
|
3539
|
+
y_pct = op.get('y_percent', 0)
|
|
3540
|
+
|
|
3541
|
+
# 转义单引号
|
|
3542
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3543
|
+
|
|
3544
|
+
if locator_type == 'text':
|
|
3545
|
+
# 文本定位(最稳定)
|
|
3546
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (文本定位)")
|
|
3547
|
+
if locator_attr == 'description':
|
|
3548
|
+
script_lines.append(f" safe_click(d, d(description='{value_escaped}'))")
|
|
3549
|
+
elif locator_attr == 'descriptionContains':
|
|
3550
|
+
script_lines.append(f" safe_click(d, d(descriptionContains='{value_escaped}'))")
|
|
3551
|
+
elif locator_attr == 'textContains':
|
|
3552
|
+
script_lines.append(f" safe_click(d, d(textContains='{value_escaped}'))")
|
|
3553
|
+
else:
|
|
3554
|
+
script_lines.append(f" safe_click(d, d(text='{value_escaped}'))")
|
|
3555
|
+
elif locator_type == 'id':
|
|
3556
|
+
# ID 定位(稳定)
|
|
3557
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (ID定位)")
|
|
3558
|
+
script_lines.append(f" safe_click(d, d(resourceId='{value_escaped}'))")
|
|
3559
|
+
elif locator_type == 'percent':
|
|
3560
|
+
# 百分比定位(跨分辨率兼容)
|
|
3561
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (百分比定位)")
|
|
3562
|
+
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
|
|
2606
3563
|
else:
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
3564
|
+
# 兼容旧格式
|
|
3565
|
+
ref = op.get('ref', '')
|
|
3566
|
+
if ref:
|
|
3567
|
+
ref_escaped = ref.replace("'", "\\'")
|
|
3568
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{ref}'")
|
|
3569
|
+
script_lines.append(f" safe_click(d, d(text='{ref_escaped}'))")
|
|
3570
|
+
else:
|
|
3571
|
+
continue
|
|
3572
|
+
|
|
3573
|
+
script_lines.append(" time.sleep(0.5)")
|
|
2610
3574
|
script_lines.append(" ")
|
|
2611
3575
|
|
|
2612
3576
|
elif action == 'input':
|
|
2613
3577
|
text = op.get('text', '')
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
if
|
|
2623
|
-
#
|
|
2624
|
-
script_lines.append(f"
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
script_lines.append(f"
|
|
2630
|
-
elif has_percent:
|
|
2631
|
-
x_pct = op['x_percent']
|
|
2632
|
-
y_pct = op['y_percent']
|
|
2633
|
-
script_lines.append(f" # 步骤{step_num}: 点击后输入 (百分比定位)")
|
|
3578
|
+
locator_type = op.get('locator_type', '')
|
|
3579
|
+
locator_value = op.get('locator_value', '')
|
|
3580
|
+
x_pct = op.get('x_percent', 0)
|
|
3581
|
+
y_pct = op.get('y_percent', 0)
|
|
3582
|
+
|
|
3583
|
+
text_escaped = text.replace("'", "\\'")
|
|
3584
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3585
|
+
|
|
3586
|
+
if locator_type == 'id':
|
|
3587
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (ID定位)")
|
|
3588
|
+
script_lines.append(f" d(resourceId='{value_escaped}').set_text('{text_escaped}')")
|
|
3589
|
+
elif locator_type == 'class':
|
|
3590
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (类名定位)")
|
|
3591
|
+
script_lines.append(f" d(className='android.widget.EditText').set_text('{text_escaped}')")
|
|
3592
|
+
elif x_pct > 0 and y_pct > 0:
|
|
3593
|
+
script_lines.append(f" # 步骤{step_num}: 点击后输入 '{text}'")
|
|
2634
3594
|
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
|
|
2635
|
-
script_lines.append(
|
|
2636
|
-
script_lines.append(f" d.send_keys('{
|
|
2637
|
-
elif has_coords:
|
|
2638
|
-
script_lines.append(f" # 步骤{step_num}: 点击坐标后输入 (⚠️ 可能不兼容其他分辨率)")
|
|
2639
|
-
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
2640
|
-
script_lines.append(f" time.sleep(0.3)")
|
|
2641
|
-
script_lines.append(f" d.send_keys('{text}')")
|
|
3595
|
+
script_lines.append(" time.sleep(0.3)")
|
|
3596
|
+
script_lines.append(f" d.send_keys('{text_escaped}')")
|
|
2642
3597
|
else:
|
|
2643
|
-
#
|
|
2644
|
-
|
|
3598
|
+
# 兼容旧格式
|
|
3599
|
+
ref = op.get('ref', '')
|
|
3600
|
+
if ref:
|
|
3601
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}'")
|
|
3602
|
+
script_lines.append(f" d(resourceId='{ref}').set_text('{text_escaped}')")
|
|
3603
|
+
else:
|
|
3604
|
+
continue
|
|
3605
|
+
|
|
2645
3606
|
script_lines.append(" time.sleep(0.5)")
|
|
2646
3607
|
script_lines.append(" ")
|
|
2647
3608
|
|
|
2648
3609
|
elif action == 'long_press':
|
|
2649
|
-
|
|
2650
|
-
|
|
3610
|
+
locator_type = op.get('locator_type', '')
|
|
3611
|
+
locator_value = op.get('locator_value', '')
|
|
3612
|
+
locator_attr = op.get('locator_attr', 'text')
|
|
3613
|
+
element_desc = op.get('element_desc', '')
|
|
2651
3614
|
duration = op.get('duration', 1.0)
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
script_lines.append(f"
|
|
2668
|
-
|
|
2669
|
-
actual_text = ref.split(':', 1)[1] if ':' in ref else ref
|
|
2670
|
-
script_lines.append(f" # 步骤{step_num}: 长按文本 '{actual_text}' (文本定位)")
|
|
2671
|
-
script_lines.append(f" d(text='{actual_text}').long_click(duration={duration})")
|
|
2672
|
-
elif has_percent:
|
|
2673
|
-
# 使用百分比
|
|
2674
|
-
x_pct = op['x_percent']
|
|
2675
|
-
y_pct = op['y_percent']
|
|
2676
|
-
desc = f" ({element})" if element else ""
|
|
2677
|
-
script_lines.append(f" # 步骤{step_num}: 长按位置{desc} (百分比定位,跨分辨率兼容)")
|
|
2678
|
-
script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
2679
|
-
elif has_coords:
|
|
2680
|
-
# 坐标兜底
|
|
2681
|
-
desc = f" ({element})" if element else ""
|
|
2682
|
-
script_lines.append(f" # 步骤{step_num}: 长按坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
2683
|
-
script_lines.append(f" d.long_click({op['x']}, {op['y']}, duration={duration})")
|
|
3615
|
+
x_pct = op.get('x_percent', 0)
|
|
3616
|
+
y_pct = op.get('y_percent', 0)
|
|
3617
|
+
|
|
3618
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3619
|
+
|
|
3620
|
+
if locator_type == 'text':
|
|
3621
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3622
|
+
if locator_attr == 'description':
|
|
3623
|
+
script_lines.append(f" d(description='{value_escaped}').long_click(duration={duration})")
|
|
3624
|
+
else:
|
|
3625
|
+
script_lines.append(f" d(text='{value_escaped}').long_click(duration={duration})")
|
|
3626
|
+
elif locator_type == 'id':
|
|
3627
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3628
|
+
script_lines.append(f" d(resourceId='{value_escaped}').long_click(duration={duration})")
|
|
3629
|
+
elif locator_type == 'percent':
|
|
3630
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3631
|
+
script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration})")
|
|
2684
3632
|
else:
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
3633
|
+
# 兼容旧格式
|
|
3634
|
+
ref = op.get('ref', '')
|
|
3635
|
+
if ref:
|
|
3636
|
+
ref_escaped = ref.replace("'", "\\'")
|
|
3637
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{ref}'")
|
|
3638
|
+
script_lines.append(f" d(text='{ref_escaped}').long_click(duration={duration})")
|
|
3639
|
+
else:
|
|
3640
|
+
continue
|
|
3641
|
+
|
|
3642
|
+
script_lines.append(" time.sleep(0.5)")
|
|
2688
3643
|
script_lines.append(" ")
|
|
2689
3644
|
|
|
2690
3645
|
elif action == 'swipe':
|
|
2691
3646
|
direction = op.get('direction', 'up')
|
|
2692
3647
|
script_lines.append(f" # 步骤{step_num}: 滑动 {direction}")
|
|
2693
|
-
script_lines.append(f" d
|
|
3648
|
+
script_lines.append(f" swipe_direction(d, '{direction}')")
|
|
2694
3649
|
script_lines.append(" time.sleep(0.5)")
|
|
2695
3650
|
script_lines.append(" ")
|
|
2696
3651
|
|
|
@@ -2705,8 +3660,16 @@ class BasicMobileToolsLite:
|
|
|
2705
3660
|
" print('✅ 测试完成')",
|
|
2706
3661
|
"",
|
|
2707
3662
|
"",
|
|
3663
|
+
"# ========== 直接运行入口 ==========",
|
|
2708
3664
|
"if __name__ == '__main__':",
|
|
2709
|
-
"
|
|
3665
|
+
" # 直接运行时,手动创建设备连接",
|
|
3666
|
+
" _d = u2.connect()",
|
|
3667
|
+
" _d.implicitly_wait(10)",
|
|
3668
|
+
" _d.app_start(PACKAGE_NAME)",
|
|
3669
|
+
" time.sleep(LAUNCH_WAIT)",
|
|
3670
|
+
" if CLOSE_AD_ON_LAUNCH:",
|
|
3671
|
+
" close_ad_if_exists(_d)",
|
|
3672
|
+
f" test_{safe_name}(_d)",
|
|
2710
3673
|
])
|
|
2711
3674
|
|
|
2712
3675
|
script = '\n'.join(script_lines)
|
|
@@ -2715,8 +3678,11 @@ class BasicMobileToolsLite:
|
|
|
2715
3678
|
output_dir = Path("tests")
|
|
2716
3679
|
output_dir.mkdir(exist_ok=True)
|
|
2717
3680
|
|
|
3681
|
+
# 确保文件名符合 pytest 规范(以 test_ 开头)
|
|
2718
3682
|
if not filename.endswith('.py'):
|
|
2719
3683
|
filename = f"{filename}.py"
|
|
3684
|
+
if not filename.startswith('test_'):
|
|
3685
|
+
filename = f"test_{filename}"
|
|
2720
3686
|
|
|
2721
3687
|
file_path = output_dir / filename
|
|
2722
3688
|
file_path.write_text(script, encoding='utf-8')
|
|
@@ -2724,7 +3690,7 @@ class BasicMobileToolsLite:
|
|
|
2724
3690
|
return {
|
|
2725
3691
|
"success": True,
|
|
2726
3692
|
"file_path": str(file_path),
|
|
2727
|
-
"message": f"✅ 脚本已生成: {file_path}",
|
|
3693
|
+
"message": f"✅ 脚本已生成: {file_path}\n💡 运行方式: pytest {file_path} -v 或 python {file_path}",
|
|
2728
3694
|
"operations_count": len(self.operation_history),
|
|
2729
3695
|
"preview": script[:500] + "..."
|
|
2730
3696
|
}
|
|
@@ -2893,10 +3859,28 @@ class BasicMobileToolsLite:
|
|
|
2893
3859
|
try:
|
|
2894
3860
|
import xml.etree.ElementTree as ET
|
|
2895
3861
|
|
|
2896
|
-
# ========== 第
|
|
3862
|
+
# ========== 第0步:先检测是否有弹窗 ==========
|
|
2897
3863
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2898
3864
|
root = ET.fromstring(xml_string)
|
|
2899
3865
|
|
|
3866
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3867
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
3868
|
+
|
|
3869
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3870
|
+
root, screen_width, screen_height
|
|
3871
|
+
)
|
|
3872
|
+
|
|
3873
|
+
# 如果没有检测到弹窗,直接返回"无弹窗"
|
|
3874
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
3875
|
+
result["success"] = True
|
|
3876
|
+
result["method"] = None
|
|
3877
|
+
result["message"] = "ℹ️ 当前页面未检测到弹窗,无需关闭"
|
|
3878
|
+
result["popup_detected"] = False
|
|
3879
|
+
result["popup_confidence"] = popup_confidence
|
|
3880
|
+
return result
|
|
3881
|
+
|
|
3882
|
+
# ========== 第1步:控件树查找关闭按钮 ==========
|
|
3883
|
+
|
|
2900
3884
|
# 关闭按钮的常见特征
|
|
2901
3885
|
close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
|
|
2902
3886
|
close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
|
|
@@ -2975,32 +3959,40 @@ class BasicMobileToolsLite:
|
|
|
2975
3959
|
cx, cy = best['center']
|
|
2976
3960
|
bounds = best['bounds']
|
|
2977
3961
|
|
|
2978
|
-
#
|
|
2979
|
-
|
|
2980
|
-
if auto_learn:
|
|
2981
|
-
pre_result = self.take_screenshot(description="关闭前", compress=False)
|
|
2982
|
-
pre_screenshot = pre_result.get("screenshot_path")
|
|
2983
|
-
|
|
2984
|
-
# 点击
|
|
2985
|
-
self.click_at_coords(cx, cy)
|
|
3962
|
+
# 点击(click_at_coords 内部已包含应用状态检查和自动返回)
|
|
3963
|
+
click_result = self.click_at_coords(cx, cy)
|
|
2986
3964
|
time.sleep(0.5)
|
|
2987
3965
|
|
|
3966
|
+
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
3967
|
+
app_check = self._check_app_switched()
|
|
3968
|
+
return_result = None
|
|
3969
|
+
|
|
3970
|
+
if app_check['switched']:
|
|
3971
|
+
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
3972
|
+
return_result = self._return_to_target_app()
|
|
3973
|
+
|
|
2988
3974
|
result["success"] = True
|
|
2989
3975
|
result["method"] = "控件树"
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
3976
|
+
msg = f"✅ 通过控件树找到关闭按钮并点击\n" \
|
|
3977
|
+
f" 位置: ({cx}, {cy})\n" \
|
|
3978
|
+
f" 原因: {best['reason']}"
|
|
2993
3979
|
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3980
|
+
if app_check['switched']:
|
|
3981
|
+
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
3982
|
+
if return_result:
|
|
3983
|
+
if return_result['success']:
|
|
3984
|
+
msg += f"\n{return_result['message']}"
|
|
3985
|
+
else:
|
|
3986
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
3987
|
+
|
|
3988
|
+
result["message"] = msg
|
|
3989
|
+
result["app_check"] = app_check
|
|
3990
|
+
result["return_to_app"] = return_result
|
|
3991
|
+
result["tip"] = "💡 建议调用 mobile_screenshot_with_som 确认弹窗是否已关闭"
|
|
3000
3992
|
|
|
3001
3993
|
return result
|
|
3002
3994
|
|
|
3003
|
-
# ========== 第2
|
|
3995
|
+
# ========== 第2步:模板匹配(自动执行,不需要 AI 介入)==========
|
|
3004
3996
|
screenshot_path = None
|
|
3005
3997
|
try:
|
|
3006
3998
|
from .template_matcher import TemplateMatcher
|
|
@@ -3020,15 +4012,30 @@ class BasicMobileToolsLite:
|
|
|
3020
4012
|
y_pct = best["percent"]["y"]
|
|
3021
4013
|
|
|
3022
4014
|
# 点击
|
|
3023
|
-
self.click_by_percent(x_pct, y_pct)
|
|
4015
|
+
click_result = self.click_by_percent(x_pct, y_pct)
|
|
3024
4016
|
time.sleep(0.5)
|
|
3025
4017
|
|
|
4018
|
+
app_check = self._check_app_switched()
|
|
4019
|
+
return_result = None
|
|
4020
|
+
|
|
4021
|
+
if app_check['switched']:
|
|
4022
|
+
return_result = self._return_to_target_app()
|
|
4023
|
+
|
|
3026
4024
|
result["success"] = True
|
|
3027
4025
|
result["method"] = "模板匹配"
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
4026
|
+
msg = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
|
|
4027
|
+
f" 模板: {best.get('template', 'unknown')}\n" \
|
|
4028
|
+
f" 置信度: {best.get('confidence', 'N/A')}%\n" \
|
|
4029
|
+
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
4030
|
+
|
|
4031
|
+
if app_check['switched']:
|
|
4032
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
4033
|
+
if return_result:
|
|
4034
|
+
msg += f"\n{return_result['message']}"
|
|
4035
|
+
|
|
4036
|
+
result["message"] = msg
|
|
4037
|
+
result["app_check"] = app_check
|
|
4038
|
+
result["return_to_app"] = return_result
|
|
3032
4039
|
return result
|
|
3033
4040
|
|
|
3034
4041
|
except ImportError:
|
|
@@ -3036,17 +4043,12 @@ class BasicMobileToolsLite:
|
|
|
3036
4043
|
except Exception:
|
|
3037
4044
|
pass # 模板匹配失败,继续下一步
|
|
3038
4045
|
|
|
3039
|
-
# ========== 第3
|
|
3040
|
-
if not screenshot_path:
|
|
3041
|
-
screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
|
|
3042
|
-
|
|
4046
|
+
# ========== 第3步:控件树和模板匹配都失败,提示 AI 使用视觉识别 ==========
|
|
3043
4047
|
result["success"] = False
|
|
4048
|
+
result["fallback"] = "vision"
|
|
3044
4049
|
result["method"] = None
|
|
3045
|
-
result["
|
|
3046
|
-
|
|
3047
|
-
"💡 找到后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
3048
|
-
result["screenshot"] = screenshot_result if not screenshot_path else {"screenshot_path": screenshot_path}
|
|
3049
|
-
result["need_ai_analysis"] = True
|
|
4050
|
+
result["popup_detected"] = True
|
|
4051
|
+
result["message"] = "⚠️ 控件树和模板匹配都未找到关闭按钮,请调用 mobile_screenshot_with_som 截图后用 click_by_som 点击"
|
|
3050
4052
|
|
|
3051
4053
|
return result
|
|
3052
4054
|
|