mobile-mcp-ai 2.5.8__py3-none-any.whl → 2.6.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,6 +8,7 @@
8
8
  - 核心功能精简
9
9
  - 保留 pytest 脚本生成
10
10
  - 支持操作历史记录
11
+ - Token 优化模式(省钱)
11
12
  """
12
13
 
13
14
  import asyncio
@@ -17,6 +18,19 @@ from pathlib import Path
17
18
  from typing import Dict, List, Optional
18
19
  from datetime import datetime
19
20
 
21
+ # Token 优化配置(只精简格式,不限制数量,确保准确度)
22
+ try:
23
+ from mobile_mcp.config import Config
24
+ TOKEN_OPTIMIZATION = Config.TOKEN_OPTIMIZATION_ENABLED
25
+ MAX_ELEMENTS = Config.MAX_ELEMENTS_RETURN
26
+ MAX_SOM_ELEMENTS = Config.MAX_SOM_ELEMENTS_RETURN
27
+ COMPACT_RESPONSE = Config.COMPACT_RESPONSE
28
+ except ImportError:
29
+ TOKEN_OPTIMIZATION = True
30
+ MAX_ELEMENTS = 0 # 0 = 不限制
31
+ MAX_SOM_ELEMENTS = 0 # 0 = 不限制
32
+ COMPACT_RESPONSE = True
33
+
20
34
 
21
35
  class BasicMobileToolsLite:
22
36
  """精简版移动端工具"""
@@ -31,6 +45,9 @@ class BasicMobileToolsLite:
31
45
 
32
46
  # 操作历史(用于生成 pytest 脚本)
33
47
  self.operation_history: List[Dict] = []
48
+
49
+ # 目标应用包名(用于监测应用跳转)
50
+ self.target_package: Optional[str] = None
34
51
 
35
52
  def _is_ios(self) -> bool:
36
53
  """判断当前是否为 iOS 平台"""
@@ -45,7 +62,7 @@ class BasicMobileToolsLite:
45
62
  return None
46
63
 
47
64
  def _record_operation(self, action: str, **kwargs):
48
- """记录操作到历史"""
65
+ """记录操作到历史(旧接口,保持兼容)"""
49
66
  record = {
50
67
  'action': action,
51
68
  'timestamp': datetime.now().isoformat(),
@@ -53,6 +70,231 @@ class BasicMobileToolsLite:
53
70
  }
54
71
  self.operation_history.append(record)
55
72
 
73
+ def _record_click(self, locator_type: str, locator_value: str,
74
+ x_percent: float = 0, y_percent: float = 0,
75
+ element_desc: str = '', locator_attr: str = ''):
76
+ """记录点击操作(标准格式)
77
+
78
+ Args:
79
+ locator_type: 定位类型 'text' | 'id' | 'percent' | 'coords'
80
+ locator_value: 定位值(文本内容、resource-id、或坐标描述)
81
+ x_percent: 百分比 X 坐标(兜底方案)
82
+ y_percent: 百分比 Y 坐标(兜底方案)
83
+ element_desc: 元素描述(用于脚本注释)
84
+ locator_attr: Android 选择器属性 'text'|'textContains'|'description'|'descriptionContains'
85
+ """
86
+ record = {
87
+ 'action': 'click',
88
+ 'timestamp': datetime.now().isoformat(),
89
+ 'locator_type': locator_type,
90
+ 'locator_value': locator_value,
91
+ 'locator_attr': locator_attr or locator_type, # 默认与 type 相同
92
+ 'x_percent': x_percent,
93
+ 'y_percent': y_percent,
94
+ 'element_desc': element_desc or locator_value,
95
+ }
96
+ self.operation_history.append(record)
97
+
98
+ def _record_long_press(self, locator_type: str, locator_value: str,
99
+ duration: float = 1.0,
100
+ x_percent: float = 0, y_percent: float = 0,
101
+ element_desc: str = '', locator_attr: str = ''):
102
+ """记录长按操作(标准格式)"""
103
+ record = {
104
+ 'action': 'long_press',
105
+ 'timestamp': datetime.now().isoformat(),
106
+ 'locator_type': locator_type,
107
+ 'locator_value': locator_value,
108
+ 'locator_attr': locator_attr or locator_type,
109
+ 'duration': duration,
110
+ 'x_percent': x_percent,
111
+ 'y_percent': y_percent,
112
+ 'element_desc': element_desc or locator_value,
113
+ }
114
+ self.operation_history.append(record)
115
+
116
+ def _record_input(self, text: str, locator_type: str = '', locator_value: str = '',
117
+ x_percent: float = 0, y_percent: float = 0):
118
+ """记录输入操作(标准格式)"""
119
+ record = {
120
+ 'action': 'input',
121
+ 'timestamp': datetime.now().isoformat(),
122
+ 'text': text,
123
+ 'locator_type': locator_type,
124
+ 'locator_value': locator_value,
125
+ 'x_percent': x_percent,
126
+ 'y_percent': y_percent,
127
+ }
128
+ self.operation_history.append(record)
129
+
130
+ def _record_swipe(self, direction: str):
131
+ """记录滑动操作"""
132
+ record = {
133
+ 'action': 'swipe',
134
+ 'timestamp': datetime.now().isoformat(),
135
+ 'direction': direction,
136
+ }
137
+ self.operation_history.append(record)
138
+
139
+ def _record_key(self, key: str):
140
+ """记录按键操作"""
141
+ record = {
142
+ 'action': 'press_key',
143
+ 'timestamp': datetime.now().isoformat(),
144
+ 'key': key,
145
+ }
146
+ self.operation_history.append(record)
147
+
148
+ def _get_current_package(self) -> Optional[str]:
149
+ """获取当前前台应用的包名/Bundle ID"""
150
+ try:
151
+ if self._is_ios():
152
+ ios_client = self._get_ios_client()
153
+ if ios_client and hasattr(ios_client, 'wda'):
154
+ app_info = ios_client.wda.session().app_current()
155
+ return app_info.get('bundleId')
156
+ else:
157
+ info = self.client.u2.app_current()
158
+ return info.get('package')
159
+ except Exception:
160
+ return None
161
+
162
+ def _check_app_switched(self) -> Dict:
163
+ """检查是否已跳出目标应用
164
+
165
+ Returns:
166
+ {
167
+ 'switched': bool, # 是否跳转
168
+ 'current_package': str, # 当前应用包名
169
+ 'target_package': str, # 目标应用包名
170
+ 'message': str # 提示信息
171
+ }
172
+ """
173
+ if not self.target_package:
174
+ return {
175
+ 'switched': False,
176
+ 'current_package': None,
177
+ 'target_package': None,
178
+ 'message': '⚠️ 未设置目标应用,无法监测应用跳转'
179
+ }
180
+
181
+ current = self._get_current_package()
182
+ if not current:
183
+ return {
184
+ 'switched': False,
185
+ 'current_package': None,
186
+ 'target_package': self.target_package,
187
+ 'message': '⚠️ 无法获取当前应用包名'
188
+ }
189
+
190
+ if current != self.target_package:
191
+ return {
192
+ 'switched': True,
193
+ 'current_package': current,
194
+ 'target_package': self.target_package,
195
+ 'message': f'⚠️ 应用已跳转!当前应用: {current},目标应用: {self.target_package}'
196
+ }
197
+
198
+ return {
199
+ 'switched': False,
200
+ 'current_package': current,
201
+ 'target_package': self.target_package,
202
+ 'message': f'✅ 仍在目标应用: {current}'
203
+ }
204
+
205
+ def _return_to_target_app(self) -> Dict:
206
+ """返回到目标应用
207
+
208
+ 策略:
209
+ 1. 先按返回键(可能关闭弹窗或返回上一页)
210
+ 2. 如果还在其他应用,启动目标应用
211
+ 3. 验证是否成功返回
212
+
213
+ Returns:
214
+ {
215
+ 'success': bool,
216
+ 'message': str,
217
+ 'method': str # 使用的返回方法
218
+ }
219
+ """
220
+ if not self.target_package:
221
+ return {
222
+ 'success': False,
223
+ 'message': '❌ 未设置目标应用,无法返回',
224
+ 'method': None
225
+ }
226
+
227
+ try:
228
+ # 先检查当前应用
229
+ current = self._get_current_package()
230
+ if not current:
231
+ return {
232
+ 'success': False,
233
+ 'message': '❌ 无法获取当前应用包名',
234
+ 'method': None
235
+ }
236
+
237
+ # 如果已经在目标应用,不需要返回
238
+ if current == self.target_package:
239
+ return {
240
+ 'success': True,
241
+ 'message': f'✅ 已在目标应用: {self.target_package}',
242
+ 'method': 'already_in_target'
243
+ }
244
+
245
+ # 策略1: 先按返回键(可能关闭弹窗或返回)
246
+ if self._is_ios():
247
+ ios_client = self._get_ios_client()
248
+ if ios_client and hasattr(ios_client, 'wda'):
249
+ # iOS 返回键
250
+ ios_client.wda.press('home') # iOS 先按 home
251
+ time.sleep(0.5)
252
+ # 然后启动目标应用
253
+ ios_client.wda.app_activate(self.target_package)
254
+ else:
255
+ return {
256
+ 'success': False,
257
+ 'message': '❌ iOS 客户端未初始化',
258
+ 'method': None
259
+ }
260
+ else:
261
+ # Android: 先按返回键
262
+ self.client.u2.press('back')
263
+ time.sleep(0.5)
264
+
265
+ # 检查是否已返回
266
+ current = self._get_current_package()
267
+ if current == self.target_package:
268
+ return {
269
+ 'success': True,
270
+ 'message': f'✅ 已返回目标应用: {self.target_package}(通过返回键)',
271
+ 'method': 'back_key'
272
+ }
273
+
274
+ # 如果还在其他应用,启动目标应用
275
+ self.client.u2.app_start(self.target_package)
276
+ time.sleep(1)
277
+
278
+ # 验证是否成功返回
279
+ current = self._get_current_package()
280
+ if current == self.target_package:
281
+ return {
282
+ 'success': True,
283
+ 'message': f'✅ 已返回目标应用: {self.target_package}',
284
+ 'method': 'app_start'
285
+ }
286
+ else:
287
+ return {
288
+ 'success': False,
289
+ 'message': f'❌ 返回失败:当前应用仍为 {current},期望 {self.target_package}',
290
+ 'method': 'app_start'
291
+ }
292
+ except Exception as e:
293
+ return {
294
+ 'success': False,
295
+ 'message': f'❌ 返回目标应用失败: {e}',
296
+ 'method': None
297
+ }
56
298
 
57
299
 
58
300
  # ==================== 截图 ====================
@@ -107,7 +349,7 @@ class BasicMobileToolsLite:
107
349
  size = ios_client.wda.window_size()
108
350
  screen_width, screen_height = size[0], size[1]
109
351
  else:
110
- return {"success": False, "message": "iOS 客户端未初始化"}
352
+ return {"success": False, "msg": "iOS未初始化"}
111
353
  else:
112
354
  self.client.u2.screenshot(str(temp_path))
113
355
  info = self.client.u2.info
@@ -158,22 +400,14 @@ class BasicMobileToolsLite:
158
400
 
159
401
  cropped_size = final_path.stat().st_size
160
402
 
403
+ # 返回结果
161
404
  return {
162
405
  "success": True,
163
406
  "screenshot_path": str(final_path),
164
- "screen_width": screen_width,
165
- "screen_height": screen_height,
166
407
  "image_width": img.width,
167
408
  "image_height": img.height,
168
409
  "crop_offset_x": crop_offset_x,
169
- "crop_offset_y": crop_offset_y,
170
- "file_size": f"{cropped_size/1024:.1f}KB",
171
- "message": f"🔍 局部截图已保存: {final_path}\n"
172
- f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
173
- f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
174
- f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
175
- f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
176
- f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
410
+ "crop_offset_y": crop_offset_y
177
411
  }
178
412
 
179
413
  # ========== 情况2:全屏压缩截图 ==========
@@ -226,24 +460,14 @@ class BasicMobileToolsLite:
226
460
  compressed_size = final_path.stat().st_size
227
461
  saved_percent = (1 - compressed_size / original_size) * 100
228
462
 
463
+ # 返回结果
229
464
  return {
230
465
  "success": True,
231
466
  "screenshot_path": str(final_path),
232
- "screen_width": screen_width,
233
- "screen_height": screen_height,
234
- "original_img_width": original_img_width, # 截图原始宽度
235
- "original_img_height": original_img_height, # 截图原始高度
236
- "image_width": image_width, # 压缩后宽度(AI 看到的)
237
- "image_height": image_height, # 压缩后高度(AI 看到的)
238
- "original_size": f"{original_size/1024:.1f}KB",
239
- "compressed_size": f"{compressed_size/1024:.1f}KB",
240
- "saved_percent": f"{saved_percent:.0f}%",
241
- "message": f"📸 截图已保存: {final_path}\n"
242
- f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
243
- f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
244
- f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
245
- f" image_width={image_width}, image_height={image_height},\n"
246
- f" original_img_width={original_img_width}, original_img_height={original_img_height}"
467
+ "image_width": image_width,
468
+ "image_height": image_height,
469
+ "original_img_width": original_img_width,
470
+ "original_img_height": original_img_height
247
471
  }
248
472
 
249
473
  # ========== 情况3:全屏不压缩截图 ==========
@@ -257,21 +481,12 @@ class BasicMobileToolsLite:
257
481
  final_path = self.screenshot_dir / filename
258
482
  temp_path.rename(final_path)
259
483
 
260
- # 不压缩时,用截图实际尺寸(可能和 screen_width 不同)
484
+ # 返回结果(不压缩时尺寸相同)
261
485
  return {
262
486
  "success": True,
263
487
  "screenshot_path": str(final_path),
264
- "screen_width": screen_width,
265
- "screen_height": screen_height,
266
- "original_img_width": img.width, # 截图实际尺寸
267
- "original_img_height": img.height,
268
- "image_width": img.width, # 未压缩,和原图一样
269
- "image_height": img.height,
270
- "file_size": f"{original_size/1024:.1f}KB",
271
- "message": f"📸 截图已保存: {final_path}\n"
272
- f"📐 截图尺寸: {img.width}x{img.height}\n"
273
- f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
274
- f"💡 未压缩,坐标可直接使用"
488
+ "image_width": img.width,
489
+ "image_height": img.height
275
490
  }
276
491
  except ImportError:
277
492
  # 如果没有 PIL,回退到原始方式(不压缩)
@@ -279,7 +494,7 @@ class BasicMobileToolsLite:
279
494
  except Exception as e:
280
495
  return {"success": False, "message": f"❌ 截图失败: {e}"}
281
496
 
282
- def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool = True) -> Dict:
497
+ def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool = False) -> Dict:
283
498
  """截图并添加网格坐标标注(用于精确定位元素)
284
499
 
285
500
  在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
@@ -311,7 +526,7 @@ class BasicMobileToolsLite:
311
526
  size = ios_client.wda.window_size()
312
527
  screen_width, screen_height = size[0], size[1]
313
528
  else:
314
- return {"success": False, "message": "iOS 客户端未初始化"}
529
+ return {"success": False, "msg": "iOS未初始化"}
315
530
  else:
316
531
  self.client.u2.screenshot(str(temp_path))
317
532
  info = self.client.u2.info
@@ -347,7 +562,7 @@ class BasicMobileToolsLite:
347
562
  # 左侧标注 Y 坐标
348
563
  draw.text((2, y + 2), str(y), fill=text_color, font=font_small)
349
564
 
350
- # 第3步:检测弹窗并标注
565
+ # 第3步:检测弹窗并标注(使用严格的置信度检测,避免误识别)
351
566
  popup_info = None
352
567
  close_positions = []
353
568
 
@@ -357,35 +572,12 @@ class BasicMobileToolsLite:
357
572
  xml_string = self.client.u2.dump_hierarchy(compressed=False)
358
573
  root = ET.fromstring(xml_string)
359
574
 
360
- # 检测弹窗区域
361
- popup_bounds = None
362
- for elem in root.iter():
363
- bounds_str = elem.attrib.get('bounds', '')
364
- class_name = elem.attrib.get('class', '')
365
-
366
- if not bounds_str:
367
- continue
368
-
369
- match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
370
- if not match:
371
- continue
372
-
373
- x1, y1, x2, y2 = map(int, match.groups())
374
- width = x2 - x1
375
- height = y2 - y1
376
- area = width * height
377
- screen_area = screen_width * screen_height
378
-
379
- is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card'])
380
- area_ratio = area / screen_area if screen_area > 0 else 0
381
- is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
382
- is_reasonable_size = 0.08 < area_ratio < 0.85
383
-
384
- if is_container and is_not_fullscreen and is_reasonable_size and y1 > 50:
385
- if popup_bounds is None or area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
386
- popup_bounds = (x1, y1, x2, y2)
575
+ # 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
576
+ popup_bounds, popup_confidence = self._detect_popup_with_confidence(
577
+ root, screen_width, screen_height
578
+ )
387
579
 
388
- if popup_bounds:
580
+ if popup_bounds and popup_confidence >= 0.6:
389
581
  px1, py1, px2, py2 = popup_bounds
390
582
  popup_width = px2 - px1
391
583
  popup_height = py2 - py1
@@ -448,26 +640,16 @@ class BasicMobileToolsLite:
448
640
  result = {
449
641
  "success": True,
450
642
  "screenshot_path": str(final_path),
451
- "screen_width": screen_width,
452
- "screen_height": screen_height,
453
643
  "image_width": img_width,
454
644
  "image_height": img_height,
455
- "grid_size": grid_size,
456
- "message": f"📸 网格截图已保存: {final_path}\n"
457
- f"📐 尺寸: {img_width}x{img_height}\n"
458
- f"📏 网格间距: {grid_size}px"
645
+ "grid_size": grid_size
459
646
  }
460
647
 
461
648
  if popup_info:
462
- result["popup_detected"] = True
463
- result["popup_bounds"] = popup_info["bounds"]
464
- result["close_button_hints"] = close_positions
465
- result["message"] += f"\n🎯 检测到弹窗: {popup_info['bounds']}"
466
- result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
467
- for pos in close_positions:
468
- result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
469
- else:
470
- result["popup_detected"] = False
649
+ result["popup"] = popup_info["bounds"]
650
+ # 只返回前3个最可能的关闭按钮位置
651
+ if close_positions:
652
+ result["close_hints"] = [(p['x'], p['y']) for p in close_positions[:3]]
471
653
 
472
654
  return result
473
655
 
@@ -504,7 +686,7 @@ class BasicMobileToolsLite:
504
686
  size = ios_client.wda.window_size()
505
687
  screen_width, screen_height = size[0], size[1]
506
688
  else:
507
- return {"success": False, "message": "iOS 客户端未初始化"}
689
+ return {"success": False, "msg": "iOS未初始化"}
508
690
  else:
509
691
  self.client.u2.screenshot(str(temp_path))
510
692
  info = self.client.u2.info
@@ -613,44 +795,24 @@ class BasicMobileToolsLite:
613
795
  'index': i + 1,
614
796
  'center': (cx, cy),
615
797
  'bounds': f"[{x1},{y1}][{x2},{y2}]",
616
- 'desc': elem['desc']
798
+ 'desc': elem['desc'],
799
+ 'text': elem.get('text', ''),
800
+ 'resource_id': elem.get('resource_id', '')
617
801
  })
618
802
 
619
- # 第3.5步:检测弹窗区域(用于标注)
803
+ # 第3.5步:检测弹窗区域(使用严格的置信度检测,避免误识别普通页面)
620
804
  popup_bounds = None
805
+ popup_confidence = 0
621
806
 
622
807
  if not self._is_ios():
623
808
  try:
624
- # 检测弹窗区域
625
- for elem in root.iter():
626
- bounds_str = elem.attrib.get('bounds', '')
627
- class_name = elem.attrib.get('class', '')
628
-
629
- if not bounds_str:
630
- continue
631
-
632
- match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
633
- if not match:
634
- continue
635
-
636
- px1, py1, px2, py2 = map(int, match.groups())
637
- p_width = px2 - px1
638
- p_height = py2 - py1
639
- p_area = p_width * p_height
640
- screen_area = screen_width * screen_height
641
-
642
- is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Frame'])
643
- area_ratio = p_area / screen_area if screen_area > 0 else 0
644
- is_not_fullscreen = (p_width < screen_width * 0.99 or p_height < screen_height * 0.95)
645
- # 放宽面积范围:5% - 95%
646
- is_reasonable_size = 0.05 < area_ratio < 0.95
647
-
648
- if is_container and is_not_fullscreen and is_reasonable_size and py1 > 30:
649
- if popup_bounds is None or p_area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
650
- popup_bounds = (px1, py1, px2, py2)
809
+ # 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
810
+ popup_bounds, popup_confidence = self._detect_popup_with_confidence(
811
+ root, screen_width, screen_height
812
+ )
651
813
 
652
814
  # 如果检测到弹窗,标注弹窗边界(不再猜测X按钮位置)
653
- if popup_bounds:
815
+ if popup_bounds and popup_confidence >= 0.6:
654
816
  px1, py1, px2, py2 = popup_bounds
655
817
 
656
818
  # 只画弹窗边框(蓝色),不再猜测X按钮位置
@@ -684,38 +846,15 @@ class BasicMobileToolsLite:
684
846
  img.save(str(final_path), "JPEG", quality=85)
685
847
  temp_path.unlink()
686
848
 
687
- # 构建元素列表文字
688
- elements_text = "\n".join([
689
- f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
690
- for e in som_elements[:15] # 只显示前15个
691
- ])
692
- if len(som_elements) > 15:
693
- elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
694
-
695
- # 构建弹窗提示文字
696
- hints_text = ""
697
- if popup_bounds:
698
- hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
699
- hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
700
- hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
701
-
849
+ # 返回结果(Token 优化:不返回 elements 列表,已存储在 self._som_elements)
702
850
  return {
703
851
  "success": True,
704
852
  "screenshot_path": str(final_path),
705
853
  "screen_width": screen_width,
706
854
  "screen_height": screen_height,
707
- "image_width": img_width,
708
- "image_height": img_height,
709
855
  "element_count": len(som_elements),
710
- "elements": som_elements,
711
856
  "popup_detected": popup_bounds is not None,
712
- "popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
713
- "message": f"📸 SoM 截图已保存: {final_path}\n"
714
- f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
715
- f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
716
- f"💡 使用方法:\n"
717
- f" - 点击标注元素:mobile_click_by_som(编号)\n"
718
- f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
857
+ "hint": "查看截图上的编号,用 click_by_som(编号) 点击"
719
858
  }
720
859
 
721
860
  except ImportError:
@@ -761,14 +900,41 @@ class BasicMobileToolsLite:
761
900
  ios_client = self._get_ios_client()
762
901
  if ios_client and hasattr(ios_client, 'wda'):
763
902
  ios_client.wda.click(cx, cy)
903
+ size = ios_client.wda.window_size()
904
+ screen_width, screen_height = size[0], size[1]
764
905
  else:
765
906
  self.client.u2.click(cx, cy)
766
-
907
+ info = self.client.u2.info
908
+ screen_width = info.get('displayWidth', 0)
909
+ screen_height = info.get('displayHeight', 0)
910
+
767
911
  time.sleep(0.3)
768
912
 
913
+ # 计算百分比坐标用于跨设备兼容
914
+ x_percent = round(cx / screen_width * 100, 1) if screen_width > 0 else 0
915
+ y_percent = round(cy / screen_height * 100, 1) if screen_height > 0 else 0
916
+
917
+ # 使用标准记录格式
918
+ # 优先使用元素的文本/描述信息,这样生成脚本时可以用文本定位
919
+ elem_text = target.get('text', '')
920
+ elem_id = target.get('resource_id', '')
921
+ elem_desc = target.get('desc', '')
922
+
923
+ if elem_text and not elem_text.startswith('['): # 排除类似 "[可点击]" 的描述
924
+ # 有文本,使用文本定位
925
+ self._record_click('text', elem_text, x_percent, y_percent,
926
+ element_desc=f"[{index}]{elem_desc}", locator_attr='text')
927
+ elif elem_id:
928
+ # 有 resource-id,使用 ID 定位
929
+ self._record_click('id', elem_id, x_percent, y_percent,
930
+ element_desc=f"[{index}]{elem_desc}")
931
+ else:
932
+ # 都没有,使用百分比定位
933
+ self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
934
+ element_desc=f"[{index}]{elem_desc}")
935
+
769
936
  return {
770
937
  "success": True,
771
- "message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
772
938
  "clicked": {
773
939
  "index": index,
774
940
  "desc": target['desc'],
@@ -802,7 +968,7 @@ class BasicMobileToolsLite:
802
968
  size = ios_client.wda.window_size()
803
969
  width, height = size[0], size[1]
804
970
  else:
805
- return {"success": False, "message": "iOS 客户端未初始化"}
971
+ return {"success": False, "msg": "iOS未初始化"}
806
972
  else:
807
973
  self.client.u2.screenshot(str(screenshot_path))
808
974
  info = self.client.u2.info
@@ -880,7 +1046,7 @@ class BasicMobileToolsLite:
880
1046
  size = ios_client.wda.window_size()
881
1047
  screen_width, screen_height = size[0], size[1]
882
1048
  else:
883
- return {"success": False, "message": "iOS 客户端未初始化"}
1049
+ return {"success": False, "msg": "iOS未初始化"}
884
1050
  else:
885
1051
  info = self.client.u2.info
886
1052
  screen_width = info.get('displayWidth', 0)
@@ -924,37 +1090,45 @@ class BasicMobileToolsLite:
924
1090
  x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
925
1091
  y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
926
1092
 
927
- # 记录操作(包含屏幕尺寸和百分比,便于脚本生成时转换)
928
- self._record_operation(
929
- 'click',
930
- x=x,
931
- y=y,
932
- x_percent=x_percent,
933
- y_percent=y_percent,
934
- screen_width=screen_width,
935
- screen_height=screen_height,
936
- ref=f"coords_{x}_{y}"
937
- )
1093
+ # 使用标准记录格式:坐标点击用百分比作为定位方式(跨分辨率兼容)
1094
+ self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
1095
+ element_desc=f"坐标({x},{y})")
1096
+
1097
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1098
+ app_check = self._check_app_switched()
1099
+ return_result = None
1100
+
1101
+ if app_check['switched']:
1102
+ # 应用已跳转,尝试返回目标应用
1103
+ return_result = self._return_to_target_app()
938
1104
 
1105
+ # 构建返回消息
939
1106
  if converted:
940
1107
  if conversion_type == "crop_offset":
941
- return {
942
- "success": True,
943
- "message": f"✅ 点击成功: ({x}, {y})\n"
944
- f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
945
- }
1108
+ msg = f"✅ 点击成功: ({x}, {y})\n" \
1109
+ f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
946
1110
  else:
947
- return {
948
- "success": True,
949
- "message": f" 点击成功: ({x}, {y})\n"
950
- f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
951
- f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
952
- }
1111
+ msg = f"✅ 点击成功: ({x}, {y})\n" \
1112
+ f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n" \
1113
+ f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
953
1114
  else:
954
- return {
955
- "success": True,
956
- "message": f"✅ 点击成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%]"
957
- }
1115
+ msg = f"✅ 点击成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%]"
1116
+
1117
+ # 如果检测到应用跳转,添加警告和返回结果
1118
+ if app_check['switched']:
1119
+ msg += f"\n{app_check['message']}"
1120
+ if return_result:
1121
+ if return_result['success']:
1122
+ msg += f"\n{return_result['message']}"
1123
+ else:
1124
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1125
+
1126
+ return {
1127
+ "success": True,
1128
+ "message": msg,
1129
+ "app_check": app_check,
1130
+ "return_to_app": return_result
1131
+ }
958
1132
  except Exception as e:
959
1133
  return {"success": False, "message": f"❌ 点击失败: {e}"}
960
1134
 
@@ -987,14 +1161,14 @@ class BasicMobileToolsLite:
987
1161
  size = ios_client.wda.window_size()
988
1162
  width, height = size[0], size[1]
989
1163
  else:
990
- return {"success": False, "message": "iOS 客户端未初始化"}
1164
+ return {"success": False, "msg": "iOS未初始化"}
991
1165
  else:
992
1166
  info = self.client.u2.info
993
1167
  width = info.get('displayWidth', 0)
994
1168
  height = info.get('displayHeight', 0)
995
1169
 
996
1170
  if width == 0 or height == 0:
997
- return {"success": False, "message": "无法获取屏幕尺寸"}
1171
+ return {"success": False, "msg": "无法获取屏幕尺寸"}
998
1172
 
999
1173
  # 第2步:百分比转像素坐标
1000
1174
  # 公式:像素 = 屏幕尺寸 × (百分比 / 100)
@@ -1009,30 +1183,29 @@ class BasicMobileToolsLite:
1009
1183
 
1010
1184
  time.sleep(0.3)
1011
1185
 
1012
- # 第4步:记录操作(同时记录百分比和像素)
1013
- self._record_operation(
1014
- 'click',
1015
- x=x,
1016
- y=y,
1017
- x_percent=x_percent,
1018
- y_percent=y_percent,
1019
- screen_width=width,
1020
- screen_height=height,
1021
- ref=f"percent_{x_percent}_{y_percent}"
1022
- )
1186
+ # 第4步:使用标准记录格式
1187
+ self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
1188
+ element_desc=f"百分比({x_percent}%,{y_percent}%)")
1023
1189
 
1024
1190
  return {
1025
1191
  "success": True,
1026
- "message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
1027
- "screen_size": {"width": width, "height": height},
1028
- "percent": {"x": x_percent, "y": y_percent},
1029
1192
  "pixel": {"x": x, "y": y}
1030
1193
  }
1031
1194
  except Exception as e:
1032
1195
  return {"success": False, "message": f"❌ 百分比点击失败: {e}"}
1033
1196
 
1034
- def click_by_text(self, text: str, timeout: float = 3.0) -> Dict:
1035
- """通过文本点击 - 先查 XML 树,再精准匹配"""
1197
+ def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None,
1198
+ verify: Optional[str] = None) -> Dict:
1199
+ """通过文本点击 - 先查 XML 树,再精准匹配
1200
+
1201
+ Args:
1202
+ text: 元素的文本内容
1203
+ timeout: 超时时间
1204
+ position: 位置信息,当有多个相同文案时使用。支持:
1205
+ - 垂直方向: "top"/"upper"/"上", "bottom"/"lower"/"下", "middle"/"center"/"中"
1206
+ - 水平方向: "left"/"左", "right"/"右", "center"/"中"
1207
+ verify: 可选,点击后验证的文本。如果指定,会检查该文本是否出现在页面上
1208
+ """
1036
1209
  try:
1037
1210
  if self._is_ios():
1038
1211
  ios_client = self._get_ios_client()
@@ -1043,19 +1216,53 @@ class BasicMobileToolsLite:
1043
1216
  if elem.exists:
1044
1217
  elem.click()
1045
1218
  time.sleep(0.3)
1046
- self._record_operation('click', element=text, ref=text)
1047
- return {"success": True, "message": f"✅ 点击成功: '{text}'"}
1048
- return {"success": False, "message": f"❌ 文本不存在: {text}"}
1219
+ self._record_click('text', text, element_desc=text, locator_attr='text')
1220
+ # 验证逻辑
1221
+ if verify:
1222
+ return self._verify_after_click(verify, ios=True)
1223
+ # 返回页面文本摘要,方便确认页面变化
1224
+ page_texts = self._get_page_texts(10)
1225
+ return {"success": True, "page_texts": page_texts}
1226
+ # 控件树找不到,提示用视觉识别
1227
+ return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
1228
+ else:
1229
+ return {"success": False, "msg": "iOS未初始化"}
1049
1230
  else:
1231
+ # 获取屏幕尺寸用于计算百分比
1232
+ screen_width, screen_height = self.client.u2.window_size()
1233
+
1050
1234
  # 🔍 先查 XML 树,找到元素及其属性
1051
- found_elem = self._find_element_in_tree(text)
1235
+ found_elem = self._find_element_in_tree(text, position=position)
1052
1236
 
1053
1237
  if found_elem:
1054
1238
  attr_type = found_elem['attr_type']
1055
1239
  attr_value = found_elem['attr_value']
1056
1240
  bounds = found_elem.get('bounds')
1057
1241
 
1058
- # 根据找到的属性类型,使用对应的选择器
1242
+ # 计算百分比坐标作为兜底
1243
+ x_pct, y_pct = 0, 0
1244
+ if bounds:
1245
+ cx = (bounds[0] + bounds[2]) // 2
1246
+ cy = (bounds[1] + bounds[3]) // 2
1247
+ x_pct = round(cx / screen_width * 100, 1)
1248
+ y_pct = round(cy / screen_height * 100, 1)
1249
+
1250
+ # 如果有位置参数,直接使用坐标点击
1251
+ if position and bounds:
1252
+ x = (bounds[0] + bounds[2]) // 2
1253
+ y = (bounds[1] + bounds[3]) // 2
1254
+ self.client.u2.click(x, y)
1255
+ time.sleep(0.3)
1256
+ self._record_click('text', attr_value, x_pct, y_pct,
1257
+ element_desc=f"{text}({position})", locator_attr=attr_type)
1258
+ # 验证逻辑
1259
+ if verify:
1260
+ return self._verify_after_click(verify)
1261
+ # 返回页面文本摘要
1262
+ page_texts = self._get_page_texts(10)
1263
+ return {"success": True, "page_texts": page_texts}
1264
+
1265
+ # 没有位置参数时,使用选择器定位
1059
1266
  if attr_type == 'text':
1060
1267
  elem = self.client.u2(text=attr_value)
1061
1268
  elif attr_type == 'textContains':
@@ -1070,33 +1277,98 @@ class BasicMobileToolsLite:
1070
1277
  if elem and elem.exists(timeout=1):
1071
1278
  elem.click()
1072
1279
  time.sleep(0.3)
1073
- self._record_operation('click', element=text, ref=f"{attr_type}:{attr_value}")
1074
- return {"success": True, "message": f"✅ 点击成功({attr_type}): '{text}'"}
1280
+ self._record_click('text', attr_value, x_pct, y_pct,
1281
+ element_desc=text, locator_attr=attr_type)
1282
+ # 验证逻辑
1283
+ if verify:
1284
+ return self._verify_after_click(verify)
1285
+ # 返回页面文本摘要
1286
+ page_texts = self._get_page_texts(10)
1287
+ return {"success": True, "page_texts": page_texts}
1075
1288
 
1076
- # 如果选择器失败,用坐标兜底
1289
+ # 选择器失败,用坐标兜底
1077
1290
  if bounds:
1078
1291
  x = (bounds[0] + bounds[2]) // 2
1079
1292
  y = (bounds[1] + bounds[3]) // 2
1080
1293
  self.client.u2.click(x, y)
1081
1294
  time.sleep(0.3)
1082
- self._record_operation('click', element=text, x=x, y=y, ref=f"coords:{x},{y}")
1083
- return {"success": True, "message": f"✅ 点击成功(坐标兜底): '{text}' @ ({x},{y})"}
1295
+ self._record_click('percent', f"{x_pct}%,{y_pct}%", x_pct, y_pct,
1296
+ element_desc=text)
1297
+ # 验证逻辑
1298
+ if verify:
1299
+ return self._verify_after_click(verify)
1300
+ # 返回页面文本摘要
1301
+ page_texts = self._get_page_texts(10)
1302
+ return {"success": True, "page_texts": page_texts}
1084
1303
 
1085
- return {"success": False, "message": f"❌ 文本不存在: {text}"}
1304
+ # 控件树找不到,提示用视觉识别
1305
+ return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
1086
1306
  except Exception as e:
1087
- return {"success": False, "message": f"❌ 点击失败: {e}"}
1307
+ return {"success": False, "msg": str(e)}
1308
+
1309
+ def _verify_after_click(self, verify_text: str, ios: bool = False, timeout: float = 2.0) -> Dict:
1310
+ """点击后验证期望文本是否出现
1311
+
1312
+ Args:
1313
+ verify_text: 期望出现的文本
1314
+ ios: 是否是 iOS 设备
1315
+ timeout: 验证超时时间
1316
+
1317
+ Returns:
1318
+ {"success": True, "verified": True/False, "hint": "..."}
1319
+ """
1320
+ time.sleep(0.5) # 等待页面更新
1321
+
1322
+ try:
1323
+ if ios:
1324
+ ios_client = self._get_ios_client()
1325
+ if ios_client and hasattr(ios_client, 'wda'):
1326
+ exists = ios_client.wda(name=verify_text).exists or \
1327
+ ios_client.wda(label=verify_text).exists
1328
+ else:
1329
+ exists = False
1330
+ else:
1331
+ # Android: 检查文本或包含文本
1332
+ exists = self.client.u2(text=verify_text).exists(timeout=timeout) or \
1333
+ self.client.u2(textContains=verify_text).exists(timeout=0.5) or \
1334
+ self.client.u2(description=verify_text).exists(timeout=0.5)
1335
+
1336
+ if exists:
1337
+ return {"success": True, "verified": True}
1338
+ else:
1339
+ # 验证失败,提示可以截图确认
1340
+ return {
1341
+ "success": True, # 点击本身成功
1342
+ "verified": False,
1343
+ "expect": verify_text,
1344
+ "hint": "验证失败,可截图确认"
1345
+ }
1346
+ except Exception as e:
1347
+ return {"success": True, "verified": False, "hint": f"验证异常: {e}"}
1088
1348
 
1089
- def _find_element_in_tree(self, text: str) -> Optional[Dict]:
1090
- """在 XML 树中查找包含指定文本的元素"""
1349
+ def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
1350
+ """在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
1351
+
1352
+ Args:
1353
+ text: 要查找的文本
1354
+ position: 位置信息,用于在有多个相同文案时筛选
1355
+ """
1091
1356
  try:
1092
1357
  xml = self.client.u2.dump_hierarchy(compressed=False)
1093
1358
  import xml.etree.ElementTree as ET
1094
1359
  root = ET.fromstring(xml)
1095
1360
 
1361
+ # 获取屏幕尺寸
1362
+ screen_width, screen_height = self.client.u2.window_size()
1363
+
1364
+ # 存储所有匹配的元素(包括不可点击的)
1365
+ matched_elements = []
1366
+
1096
1367
  for elem in root.iter():
1097
1368
  elem_text = elem.attrib.get('text', '')
1098
1369
  elem_desc = elem.attrib.get('content-desc', '')
1099
1370
  bounds_str = elem.attrib.get('bounds', '')
1371
+ clickable = elem.attrib.get('clickable', 'false').lower() == 'true'
1100
1372
 
1101
1373
  # 解析 bounds
1102
1374
  bounds = None
@@ -1106,36 +1378,113 @@ class BasicMobileToolsLite:
1106
1378
  if len(match) == 4:
1107
1379
  bounds = [int(x) for x in match]
1108
1380
 
1381
+ # 判断是否匹配
1382
+ is_match = False
1383
+ attr_type = None
1384
+ attr_value = None
1385
+
1109
1386
  # 精确匹配 text
1110
1387
  if elem_text == text:
1111
- return {'attr_type': 'text', 'attr_value': text, 'bounds': bounds}
1112
-
1388
+ is_match = True
1389
+ attr_type = 'text'
1390
+ attr_value = text
1113
1391
  # 精确匹配 content-desc
1114
- if elem_desc == text:
1115
- return {'attr_type': 'description', 'attr_value': text, 'bounds': bounds}
1116
-
1392
+ elif elem_desc == text:
1393
+ is_match = True
1394
+ attr_type = 'description'
1395
+ attr_value = text
1117
1396
  # 模糊匹配 text
1118
- if text in elem_text:
1119
- return {'attr_type': 'textContains', 'attr_value': text, 'bounds': bounds}
1120
-
1397
+ elif text in elem_text:
1398
+ is_match = True
1399
+ attr_type = 'textContains'
1400
+ attr_value = text
1121
1401
  # 模糊匹配 content-desc
1122
- if text in elem_desc:
1123
- return {'attr_type': 'descriptionContains', 'attr_value': text, 'bounds': bounds}
1402
+ elif text in elem_desc:
1403
+ is_match = True
1404
+ attr_type = 'descriptionContains'
1405
+ attr_value = text
1406
+
1407
+ if is_match and bounds:
1408
+ # 计算元素的中心点坐标
1409
+ center_x = (bounds[0] + bounds[2]) / 2
1410
+ center_y = (bounds[1] + bounds[3]) / 2
1411
+
1412
+ matched_elements.append({
1413
+ 'attr_type': attr_type,
1414
+ 'attr_value': attr_value,
1415
+ 'bounds': bounds,
1416
+ 'clickable': clickable,
1417
+ 'center_x': center_x,
1418
+ 'center_y': center_y
1419
+ })
1420
+
1421
+ if not matched_elements:
1422
+ return None
1423
+
1424
+ # 如果有位置信息,根据位置筛选
1425
+ if position and len(matched_elements) > 1:
1426
+ position_lower = position.lower()
1427
+
1428
+ # 根据位置信息排序
1429
+ if position_lower in ['top', 'upper', '上', '上方']:
1430
+ # 选择 y 坐标最小的(最上面的)
1431
+ matched_elements = sorted(matched_elements, key=lambda x: x['center_y'])
1432
+ elif position_lower in ['bottom', 'lower', '下', '下方', '底部']:
1433
+ # 选择 y 坐标最大的(最下面的)
1434
+ matched_elements = sorted(matched_elements, key=lambda x: x['center_y'], reverse=True)
1435
+ elif position_lower in ['left', '左', '左侧']:
1436
+ # 选择 x 坐标最小的(最左边的)
1437
+ matched_elements = sorted(matched_elements, key=lambda x: x['center_x'])
1438
+ elif position_lower in ['right', '右', '右侧']:
1439
+ # 选择 x 坐标最大的(最右边的)
1440
+ matched_elements = sorted(matched_elements, key=lambda x: x['center_x'], reverse=True)
1441
+ elif position_lower in ['middle', 'center', '中', '中间']:
1442
+ # 选择最接近屏幕中心的
1443
+ screen_mid_x = screen_width / 2
1444
+ screen_mid_y = screen_height / 2
1445
+ matched_elements = sorted(
1446
+ matched_elements,
1447
+ key=lambda x: abs(x['center_x'] - screen_mid_x) + abs(x['center_y'] - screen_mid_y)
1448
+ )
1449
+
1450
+ # 如果有位置信息,优先返回排序后的第一个元素(最符合位置要求的)
1451
+ # 如果没有位置信息,优先返回可点击的元素
1452
+ if position and matched_elements:
1453
+ # 有位置信息时,直接返回排序后的第一个(最符合位置要求的)
1454
+ first_match = matched_elements[0]
1455
+ return {
1456
+ 'attr_type': first_match['attr_type'],
1457
+ 'attr_value': first_match['attr_value'],
1458
+ 'bounds': first_match['bounds']
1459
+ }
1460
+
1461
+ # 没有位置信息时,优先返回可点击的元素
1462
+ for match in matched_elements:
1463
+ if match['clickable']:
1464
+ return {
1465
+ 'attr_type': match['attr_type'],
1466
+ 'attr_value': match['attr_value'],
1467
+ 'bounds': match['bounds']
1468
+ }
1469
+
1470
+ # 如果没有可点击的元素,直接返回第一个匹配元素的 bounds(使用坐标点击)
1471
+ if matched_elements:
1472
+ first_match = matched_elements[0]
1473
+ return {
1474
+ 'attr_type': first_match['attr_type'],
1475
+ 'attr_value': first_match['attr_value'],
1476
+ 'bounds': first_match['bounds']
1477
+ }
1124
1478
 
1125
1479
  return None
1126
- except Exception:
1480
+ except Exception as e:
1481
+ import traceback
1482
+ traceback.print_exc()
1127
1483
  return None
1128
1484
 
1129
1485
  def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
1130
- """通过 resource-id 点击(支持点击第 N 个元素)
1131
-
1132
- Args:
1133
- resource_id: 元素的 resource-id
1134
- index: 第几个元素(从 0 开始),默认 0 表示第一个
1135
- """
1486
+ """通过 resource-id 点击"""
1136
1487
  try:
1137
- index_desc = f"[{index}]" if index > 0 else ""
1138
-
1139
1488
  if self._is_ios():
1140
1489
  ios_client = self._get_ios_client()
1141
1490
  if ios_client and hasattr(ios_client, 'wda'):
@@ -1143,31 +1492,31 @@ class BasicMobileToolsLite:
1143
1492
  if not elem.exists:
1144
1493
  elem = ios_client.wda(name=resource_id)
1145
1494
  if elem.exists:
1146
- # 获取所有匹配的元素
1147
1495
  elements = elem.find_elements()
1148
1496
  if index < len(elements):
1149
1497
  elements[index].click()
1150
1498
  time.sleep(0.3)
1151
- self._record_operation('click', element=f"{resource_id}{index_desc}", ref=resource_id, index=index)
1152
- return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}"}
1499
+ self._record_click('id', resource_id, element_desc=resource_id)
1500
+ return {"success": True}
1153
1501
  else:
1154
- return {"success": False, "message": f"❌ 索引超出范围: 找到 {len(elements)} 个元素,但请求索引 {index}"}
1155
- return {"success": False, "message": f" 元素不存在: {resource_id}"}
1502
+ return {"success": False, "msg": f"索引{index}超出范围(共{len(elements)}个)"}
1503
+ return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
1504
+ else:
1505
+ return {"success": False, "msg": "iOS未初始化"}
1156
1506
  else:
1157
1507
  elem = self.client.u2(resourceId=resource_id)
1158
1508
  if elem.exists(timeout=0.5):
1159
- # 获取匹配元素数量
1160
1509
  count = elem.count
1161
1510
  if index < count:
1162
1511
  elem[index].click()
1163
1512
  time.sleep(0.3)
1164
- self._record_operation('click', element=f"{resource_id}{index_desc}", ref=resource_id, index=index)
1165
- return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}" + (f" (共 {count} 个)" if count > 1 else "")}
1513
+ self._record_click('id', resource_id, element_desc=resource_id)
1514
+ return {"success": True}
1166
1515
  else:
1167
- return {"success": False, "message": f"❌ 索引超出范围: 找到 {count} 个元素,但请求索引 {index}"}
1168
- return {"success": False, "message": f" 元素不存在: {resource_id}"}
1516
+ return {"success": False, "msg": f"索引{index}超出范围(共{count}个)"}
1517
+ return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
1169
1518
  except Exception as e:
1170
- return {"success": False, "message": f"❌ 点击失败: {e}"}
1519
+ return {"success": False, "msg": str(e)}
1171
1520
 
1172
1521
  # ==================== 长按操作 ====================
1173
1522
 
@@ -1201,7 +1550,7 @@ class BasicMobileToolsLite:
1201
1550
  size = ios_client.wda.window_size()
1202
1551
  screen_width, screen_height = size[0], size[1]
1203
1552
  else:
1204
- return {"success": False, "message": "iOS 客户端未初始化"}
1553
+ return {"success": False, "msg": "iOS未初始化"}
1205
1554
  else:
1206
1555
  info = self.client.u2.info
1207
1556
  screen_width = info.get('displayWidth', 0)
@@ -1248,38 +1597,17 @@ class BasicMobileToolsLite:
1248
1597
  x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
1249
1598
  y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
1250
1599
 
1251
- # 记录操作
1252
- self._record_operation(
1253
- 'long_press',
1254
- x=x,
1255
- y=y,
1256
- x_percent=x_percent,
1257
- y_percent=y_percent,
1258
- duration=duration,
1259
- screen_width=screen_width,
1260
- screen_height=screen_height,
1261
- ref=f"coords_{x}_{y}"
1262
- )
1600
+ # 使用标准记录格式
1601
+ self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
1602
+ x_percent, y_percent, element_desc=f"坐标({x},{y})")
1263
1603
 
1264
1604
  if converted:
1265
1605
  if conversion_type == "crop_offset":
1266
- return {
1267
- "success": True,
1268
- "message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
1269
- f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
1270
- }
1606
+ return {"success": True}
1271
1607
  else:
1272
- return {
1273
- "success": True,
1274
- "message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
1275
- f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
1276
- f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
1277
- }
1608
+ return {"success": True}
1278
1609
  else:
1279
- return {
1280
- "success": True,
1281
- "message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
1282
- }
1610
+ return {"success": True}
1283
1611
  except Exception as e:
1284
1612
  return {"success": False, "message": f"❌ 长按失败: {e}"}
1285
1613
 
@@ -1308,14 +1636,14 @@ class BasicMobileToolsLite:
1308
1636
  size = ios_client.wda.window_size()
1309
1637
  width, height = size[0], size[1]
1310
1638
  else:
1311
- return {"success": False, "message": "iOS 客户端未初始化"}
1639
+ return {"success": False, "msg": "iOS未初始化"}
1312
1640
  else:
1313
1641
  info = self.client.u2.info
1314
1642
  width = info.get('displayWidth', 0)
1315
1643
  height = info.get('displayHeight', 0)
1316
1644
 
1317
1645
  if width == 0 or height == 0:
1318
- return {"success": False, "message": "无法获取屏幕尺寸"}
1646
+ return {"success": False, "msg": "无法获取屏幕尺寸"}
1319
1647
 
1320
1648
  # 第2步:百分比转像素坐标
1321
1649
  x = int(width * x_percent / 100)
@@ -1333,26 +1661,11 @@ class BasicMobileToolsLite:
1333
1661
 
1334
1662
  time.sleep(0.3)
1335
1663
 
1336
- # 第4步:记录操作
1337
- self._record_operation(
1338
- 'long_press',
1339
- x=x,
1340
- y=y,
1341
- x_percent=x_percent,
1342
- y_percent=y_percent,
1343
- duration=duration,
1344
- screen_width=width,
1345
- screen_height=height,
1346
- ref=f"percent_{x_percent}_{y_percent}"
1347
- )
1664
+ # 第4步:使用标准记录格式
1665
+ self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
1666
+ x_percent, y_percent, element_desc=f"百分比({x_percent}%,{y_percent}%)")
1348
1667
 
1349
- return {
1350
- "success": True,
1351
- "message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
1352
- "screen_size": {"width": width, "height": height},
1353
- "percent": {"x": x_percent, "y": y_percent},
1354
- "pixel": {"x": x, "y": y},
1355
- "duration": duration
1668
+ return {"success": True
1356
1669
  }
1357
1670
  except Exception as e:
1358
1671
  return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
@@ -1381,10 +1694,13 @@ class BasicMobileToolsLite:
1381
1694
  else:
1382
1695
  ios_client.wda.swipe(x, y, x, y, duration=duration)
1383
1696
  time.sleep(0.3)
1384
- self._record_operation('long_press', element=text, duration=duration, ref=text)
1385
- return {"success": True, "message": f"✅ 长按成功: '{text}' 持续 {duration}s"}
1386
- return {"success": False, "message": f"❌ 文本不存在: {text}"}
1697
+ self._record_long_press('text', text, duration, element_desc=text, locator_attr='text')
1698
+ return {"success": True}
1699
+ return {"success": False, "msg": f"未找到'{text}'"}
1387
1700
  else:
1701
+ # 获取屏幕尺寸用于计算百分比
1702
+ screen_width, screen_height = self.client.u2.window_size()
1703
+
1388
1704
  # 先查 XML 树,找到元素
1389
1705
  found_elem = self._find_element_in_tree(text)
1390
1706
 
@@ -1393,6 +1709,14 @@ class BasicMobileToolsLite:
1393
1709
  attr_value = found_elem['attr_value']
1394
1710
  bounds = found_elem.get('bounds')
1395
1711
 
1712
+ # 计算百分比坐标作为兜底
1713
+ x_pct, y_pct = 0, 0
1714
+ if bounds:
1715
+ cx = (bounds[0] + bounds[2]) // 2
1716
+ cy = (bounds[1] + bounds[3]) // 2
1717
+ x_pct = round(cx / screen_width * 100, 1)
1718
+ y_pct = round(cy / screen_height * 100, 1)
1719
+
1396
1720
  # 根据找到的属性类型,使用对应的选择器
1397
1721
  if attr_type == 'text':
1398
1722
  elem = self.client.u2(text=attr_value)
@@ -1408,8 +1732,9 @@ class BasicMobileToolsLite:
1408
1732
  if elem and elem.exists(timeout=1):
1409
1733
  elem.long_click(duration=duration)
1410
1734
  time.sleep(0.3)
1411
- self._record_operation('long_press', element=text, duration=duration, ref=f"{attr_type}:{attr_value}")
1412
- return {"success": True, "message": f"✅ 长按成功({attr_type}): '{text}' 持续 {duration}s"}
1735
+ self._record_long_press('text', attr_value, duration, x_pct, y_pct,
1736
+ element_desc=text, locator_attr=attr_type)
1737
+ return {"success": True}
1413
1738
 
1414
1739
  # 如果选择器失败,用坐标兜底
1415
1740
  if bounds:
@@ -1417,10 +1742,11 @@ class BasicMobileToolsLite:
1417
1742
  y = (bounds[1] + bounds[3]) // 2
1418
1743
  self.client.u2.long_click(x, y, duration=duration)
1419
1744
  time.sleep(0.3)
1420
- self._record_operation('long_press', element=text, x=x, y=y, duration=duration, ref=f"coords:{x},{y}")
1421
- return {"success": True, "message": f"✅ 长按成功(坐标兜底): '{text}' @ ({x},{y}) 持续 {duration}s"}
1745
+ self._record_long_press('percent', f"{x_pct}%,{y_pct}%", duration, x_pct, y_pct,
1746
+ element_desc=text)
1747
+ return {"success": True}
1422
1748
 
1423
- return {"success": False, "message": f"❌ 文本不存在: {text}"}
1749
+ return {"success": False, "msg": f"未找到'{text}'"}
1424
1750
  except Exception as e:
1425
1751
  return {"success": False, "message": f"❌ 长按失败: {e}"}
1426
1752
 
@@ -1447,17 +1773,17 @@ class BasicMobileToolsLite:
1447
1773
  else:
1448
1774
  ios_client.wda.swipe(x, y, x, y, duration=duration)
1449
1775
  time.sleep(0.3)
1450
- self._record_operation('long_press', element=resource_id, duration=duration, ref=resource_id)
1451
- return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
1452
- return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
1776
+ self._record_long_press('id', resource_id, duration, element_desc=resource_id)
1777
+ return {"success": True}
1778
+ return {"success": False, "msg": f"未找到'{resource_id}'"}
1453
1779
  else:
1454
1780
  elem = self.client.u2(resourceId=resource_id)
1455
1781
  if elem.exists(timeout=0.5):
1456
1782
  elem.long_click(duration=duration)
1457
1783
  time.sleep(0.3)
1458
- self._record_operation('long_press', element=resource_id, duration=duration, ref=resource_id)
1784
+ self._record_long_press('id', resource_id, duration, element_desc=resource_id)
1459
1785
  return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
1460
- return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
1786
+ return {"success": False, "msg": f"未找到'{resource_id}'"}
1461
1787
  except Exception as e:
1462
1788
  return {"success": False, "message": f"❌ 长按失败: {e}"}
1463
1789
 
@@ -1482,8 +1808,29 @@ class BasicMobileToolsLite:
1482
1808
  if elem.exists:
1483
1809
  elem.set_text(text)
1484
1810
  time.sleep(0.3)
1485
- self._record_operation('input', element=resource_id, ref=resource_id, text=text)
1486
- return {"success": True, "message": f"✅ 输入成功: '{text}'"}
1811
+ self._record_input(text, 'id', resource_id)
1812
+
1813
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1814
+ app_check = self._check_app_switched()
1815
+ return_result = None
1816
+ if app_check['switched']:
1817
+ return_result = self._return_to_target_app()
1818
+
1819
+ msg = f"✅ 输入成功: '{text}'"
1820
+ if app_check['switched']:
1821
+ msg += f"\n{app_check['message']}"
1822
+ if return_result:
1823
+ if return_result['success']:
1824
+ msg += f"\n{return_result['message']}"
1825
+ else:
1826
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1827
+
1828
+ return {
1829
+ "success": True,
1830
+ "message": msg,
1831
+ "app_check": app_check,
1832
+ "return_to_app": return_result
1833
+ }
1487
1834
  return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
1488
1835
  else:
1489
1836
  elements = self.client.u2(resourceId=resource_id)
@@ -1496,8 +1843,29 @@ class BasicMobileToolsLite:
1496
1843
  if count == 1:
1497
1844
  elements.set_text(text)
1498
1845
  time.sleep(0.3)
1499
- self._record_operation('input', element=resource_id, ref=resource_id, text=text)
1500
- return {"success": True, "message": f"✅ 输入成功: '{text}'"}
1846
+ self._record_input(text, 'id', resource_id)
1847
+
1848
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1849
+ app_check = self._check_app_switched()
1850
+ return_result = None
1851
+ if app_check['switched']:
1852
+ return_result = self._return_to_target_app()
1853
+
1854
+ msg = f"✅ 输入成功: '{text}'"
1855
+ if app_check['switched']:
1856
+ msg += f"\n{app_check['message']}"
1857
+ if return_result:
1858
+ if return_result['success']:
1859
+ msg += f"\n{return_result['message']}"
1860
+ else:
1861
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1862
+
1863
+ return {
1864
+ "success": True,
1865
+ "message": msg,
1866
+ "app_check": app_check,
1867
+ "return_to_app": return_result
1868
+ }
1501
1869
 
1502
1870
  # 多个相同 ID(<=5个),尝试智能选择
1503
1871
  if count <= 5:
@@ -1509,15 +1877,57 @@ class BasicMobileToolsLite:
1509
1877
  if info.get('editable') or info.get('focusable'):
1510
1878
  elem.set_text(text)
1511
1879
  time.sleep(0.3)
1512
- self._record_operation('input', element=resource_id, ref=resource_id, text=text)
1513
- return {"success": True, "message": f"✅ 输入成功: '{text}'"}
1880
+ self._record_input(text, 'id', resource_id)
1881
+
1882
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1883
+ app_check = self._check_app_switched()
1884
+ return_result = None
1885
+ if app_check['switched']:
1886
+ return_result = self._return_to_target_app()
1887
+
1888
+ msg = f"✅ 输入成功: '{text}'"
1889
+ if app_check['switched']:
1890
+ msg += f"\n{app_check['message']}"
1891
+ if return_result:
1892
+ if return_result['success']:
1893
+ msg += f"\n{return_result['message']}"
1894
+ else:
1895
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1896
+
1897
+ return {
1898
+ "success": True,
1899
+ "message": msg,
1900
+ "app_check": app_check,
1901
+ "return_to_app": return_result
1902
+ }
1514
1903
  except:
1515
1904
  continue
1516
1905
  # 没找到可编辑的,用第一个
1517
1906
  elements[0].set_text(text)
1518
1907
  time.sleep(0.3)
1519
- self._record_operation('input', element=resource_id, ref=resource_id, text=text)
1520
- return {"success": True, "message": f"✅ 输入成功: '{text}'"}
1908
+ self._record_input(text, 'id', resource_id)
1909
+
1910
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1911
+ app_check = self._check_app_switched()
1912
+ return_result = None
1913
+ if app_check['switched']:
1914
+ return_result = self._return_to_target_app()
1915
+
1916
+ msg = f"✅ 输入成功: '{text}'"
1917
+ if app_check['switched']:
1918
+ msg += f"\n{app_check['message']}"
1919
+ if return_result:
1920
+ if return_result['success']:
1921
+ msg += f"\n{return_result['message']}"
1922
+ else:
1923
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1924
+
1925
+ return {
1926
+ "success": True,
1927
+ "message": msg,
1928
+ "app_check": app_check,
1929
+ "return_to_app": return_result
1930
+ }
1521
1931
 
1522
1932
  # ID 不可靠(不存在或太多),改用 EditText 类型定位
1523
1933
  edit_texts = self.client.u2(className='android.widget.EditText')
@@ -1526,10 +1936,31 @@ class BasicMobileToolsLite:
1526
1936
  if et_count == 1:
1527
1937
  edit_texts.set_text(text)
1528
1938
  time.sleep(0.3)
1529
- self._record_operation('input', element='EditText', ref='EditText', text=text)
1530
- return {"success": True, "message": f"✅ 输入成功: '{text}' (通过 EditText 定位)"}
1531
-
1532
- # 多个 EditText,选择最靠上的
1939
+ self._record_input(text, 'class', 'EditText')
1940
+
1941
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1942
+ app_check = self._check_app_switched()
1943
+ return_result = None
1944
+ if app_check['switched']:
1945
+ return_result = self._return_to_target_app()
1946
+
1947
+ msg = f"✅ 输入成功: '{text}' (通过 EditText 定位)"
1948
+ if app_check['switched']:
1949
+ msg += f"\n{app_check['message']}"
1950
+ if return_result:
1951
+ if return_result['success']:
1952
+ msg += f"\n{return_result['message']}"
1953
+ else:
1954
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1955
+
1956
+ return {
1957
+ "success": True,
1958
+ "message": msg,
1959
+ "app_check": app_check,
1960
+ "return_to_app": return_result
1961
+ }
1962
+
1963
+ # 多个 EditText,选择最靠上的
1533
1964
  best_elem = None
1534
1965
  min_top = 9999
1535
1966
  for i in range(et_count):
@@ -1545,8 +1976,29 @@ class BasicMobileToolsLite:
1545
1976
  if best_elem:
1546
1977
  best_elem.set_text(text)
1547
1978
  time.sleep(0.3)
1548
- self._record_operation('input', element='EditText', ref='EditText', text=text)
1549
- return {"success": True, "message": f"✅ 输入成功: '{text}' (通过 EditText 定位,选择最顶部的)"}
1979
+ self._record_input(text, 'class', 'EditText')
1980
+
1981
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
1982
+ app_check = self._check_app_switched()
1983
+ return_result = None
1984
+ if app_check['switched']:
1985
+ return_result = self._return_to_target_app()
1986
+
1987
+ msg = f"✅ 输入成功: '{text}' (通过 EditText 定位,选择最顶部的)"
1988
+ if app_check['switched']:
1989
+ msg += f"\n{app_check['message']}"
1990
+ if return_result:
1991
+ if return_result['success']:
1992
+ msg += f"\n{return_result['message']}"
1993
+ else:
1994
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
1995
+
1996
+ return {
1997
+ "success": True,
1998
+ "message": msg,
1999
+ "app_check": app_check,
2000
+ "return_to_app": return_result
2001
+ }
1550
2002
 
1551
2003
  return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
1552
2004
 
@@ -1588,17 +2040,32 @@ class BasicMobileToolsLite:
1588
2040
  x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
1589
2041
  y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
1590
2042
 
1591
- self._record_operation(
1592
- 'input',
1593
- x=x,
1594
- y=y,
1595
- x_percent=x_percent,
1596
- y_percent=y_percent,
1597
- ref=f"coords_{x}_{y}",
1598
- text=text
1599
- )
2043
+ # 使用标准记录格式
2044
+ self._record_input(text, 'percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent)
1600
2045
 
1601
- return {"success": True, "message": f"✅ 输入成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%] -> '{text}'"}
2046
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
2047
+ app_check = self._check_app_switched()
2048
+ return_result = None
2049
+
2050
+ if app_check['switched']:
2051
+ # 应用已跳转,尝试返回目标应用
2052
+ return_result = self._return_to_target_app()
2053
+
2054
+ msg = f"✅ 输入成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%] -> '{text}'"
2055
+ if app_check['switched']:
2056
+ msg += f"\n{app_check['message']}"
2057
+ if return_result:
2058
+ if return_result['success']:
2059
+ msg += f"\n{return_result['message']}"
2060
+ else:
2061
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
2062
+
2063
+ return {
2064
+ "success": True,
2065
+ "message": msg,
2066
+ "app_check": app_check,
2067
+ "return_to_app": return_result
2068
+ }
1602
2069
  except Exception as e:
1603
2070
  return {"success": False, "message": f"❌ 输入失败: {e}"}
1604
2071
 
@@ -1619,7 +2086,7 @@ class BasicMobileToolsLite:
1619
2086
  size = ios_client.wda.window_size()
1620
2087
  width, height = size[0], size[1]
1621
2088
  else:
1622
- return {"success": False, "message": "iOS 客户端未初始化"}
2089
+ return {"success": False, "msg": "iOS未初始化"}
1623
2090
  else:
1624
2091
  width, height = self.client.u2.window_size()
1625
2092
 
@@ -1657,13 +2124,16 @@ class BasicMobileToolsLite:
1657
2124
  else:
1658
2125
  self.client.u2.swipe(x1, y1, x2, y2, duration=0.5)
1659
2126
 
1660
- # 记录操作信息
1661
- record_info = {'direction': direction}
1662
- if y is not None:
1663
- record_info['y'] = y
1664
- if y_percent is not None:
1665
- record_info['y_percent'] = y_percent
1666
- self._record_operation('swipe', **record_info)
2127
+ # 使用标准记录格式
2128
+ self._record_swipe(direction)
2129
+
2130
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
2131
+ app_check = self._check_app_switched()
2132
+ return_result = None
2133
+
2134
+ if app_check['switched']:
2135
+ # 应用已跳转,尝试返回目标应用
2136
+ return_result = self._return_to_target_app()
1667
2137
 
1668
2138
  # 构建返回消息
1669
2139
  msg = f"✅ 滑动成功: {direction}"
@@ -1673,7 +2143,21 @@ class BasicMobileToolsLite:
1673
2143
  elif y is not None:
1674
2144
  msg += f" (高度: {y}px)"
1675
2145
 
1676
- return {"success": True, "message": msg}
2146
+ # 如果检测到应用跳转,添加警告和返回结果
2147
+ if app_check['switched']:
2148
+ msg += f"\n{app_check['message']}"
2149
+ if return_result:
2150
+ if return_result['success']:
2151
+ msg += f"\n{return_result['message']}"
2152
+ else:
2153
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
2154
+
2155
+ return {
2156
+ "success": True,
2157
+ "message": msg,
2158
+ "app_check": app_check,
2159
+ "return_to_app": return_result
2160
+ }
1677
2161
  except Exception as e:
1678
2162
  return {"success": False, "message": f"❌ 滑动失败: {e}"}
1679
2163
 
@@ -1698,22 +2182,22 @@ class BasicMobileToolsLite:
1698
2182
  ios_client.wda.send_keys('\n')
1699
2183
  elif ios_key == 'home':
1700
2184
  ios_client.wda.home()
1701
- return {"success": True, "message": f"✅ 按键成功: {key}"}
1702
- return {"success": False, "message": f"iOS 不支持: {key}"}
2185
+ return {"success": True}
2186
+ return {"success": False, "msg": f"iOS不支持{key}"}
1703
2187
  else:
1704
2188
  keycode = key_map.get(key.lower())
1705
2189
  if keycode:
1706
2190
  self.client.u2.shell(f'input keyevent {keycode}')
1707
- self._record_operation('press_key', key=key)
1708
- return {"success": True, "message": f"✅ 按键成功: {key}"}
1709
- return {"success": False, "message": f"❌ 不支持的按键: {key}"}
2191
+ self._record_key(key)
2192
+ return {"success": True}
2193
+ return {"success": False, "msg": f"不支持按键{key}"}
1710
2194
  except Exception as e:
1711
2195
  return {"success": False, "message": f"❌ 按键失败: {e}"}
1712
2196
 
1713
2197
  def wait(self, seconds: float) -> Dict:
1714
2198
  """等待指定时间"""
1715
2199
  time.sleep(seconds)
1716
- return {"success": True, "message": f"✅ 已等待 {seconds} 秒"}
2200
+ return {"success": True}
1717
2201
 
1718
2202
  # ==================== 应用管理 ====================
1719
2203
 
@@ -1729,12 +2213,20 @@ class BasicMobileToolsLite:
1729
2213
 
1730
2214
  await asyncio.sleep(2)
1731
2215
 
2216
+ # 记录目标应用包名(用于后续监测应用跳转)
2217
+ self.target_package = package_name
2218
+
2219
+ # 验证是否成功启动到目标应用
2220
+ current = self._get_current_package()
2221
+ if current and current != package_name:
2222
+ return {
2223
+ "success": False,
2224
+ "message": f"❌ 启动失败:当前应用为 {current},期望 {package_name}"
2225
+ }
2226
+
1732
2227
  self._record_operation('launch_app', package_name=package_name)
1733
2228
 
1734
- return {
1735
- "success": True,
1736
- "message": f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载"
1737
- }
2229
+ return {"success": True}
1738
2230
  except Exception as e:
1739
2231
  return {"success": False, "message": f"❌ 启动失败: {e}"}
1740
2232
 
@@ -1747,9 +2239,9 @@ class BasicMobileToolsLite:
1747
2239
  ios_client.wda.app_terminate(package_name)
1748
2240
  else:
1749
2241
  self.client.u2.app_stop(package_name)
1750
- return {"success": True, "message": f"✅ 已终止: {package_name}"}
2242
+ return {"success": True}
1751
2243
  except Exception as e:
1752
- return {"success": False, "message": f"❌ 终止失败: {e}"}
2244
+ return {"success": False, "msg": str(e)}
1753
2245
 
1754
2246
  def list_apps(self, filter_keyword: str = "") -> Dict:
1755
2247
  """列出已安装应用"""
@@ -1823,7 +2315,7 @@ class BasicMobileToolsLite:
1823
2315
  # ==================== 辅助工具 ====================
1824
2316
 
1825
2317
  def list_elements(self) -> List[Dict]:
1826
- """列出页面元素"""
2318
+ """列出页面元素(已优化:过滤排版容器,保留功能控件)"""
1827
2319
  try:
1828
2320
  if self._is_ios():
1829
2321
  ios_client = self._get_ios_client()
@@ -1834,20 +2326,268 @@ class BasicMobileToolsLite:
1834
2326
  xml_string = self.client.u2.dump_hierarchy(compressed=False)
1835
2327
  elements = self.client.xml_parser.parse(xml_string)
1836
2328
 
2329
+ # 功能控件类型(需要保留)
2330
+ FUNCTIONAL_WIDGETS = {
2331
+ 'TextView', 'Text', 'Label', # 文本类
2332
+ 'ImageView', 'Image', 'ImageButton', # 图片类
2333
+ 'Button', 'CheckBox', 'RadioButton', 'Switch', # 交互类
2334
+ 'SeekBar', 'ProgressBar', 'RatingBar', # 滑动/进度类
2335
+ 'EditText', 'TextInput', # 输入类
2336
+ 'VideoView', 'WebView', # 特殊功能类
2337
+ 'RecyclerView', 'ListView', 'GridView', # 列表类
2338
+ 'ScrollView', 'NestedScrollView', # 滚动容器(有实际功能)
2339
+ }
2340
+
2341
+ # 容器控件类型(需要过滤,除非有业务ID)
2342
+ CONTAINER_WIDGETS = {
2343
+ 'FrameLayout', 'LinearLayout', 'RelativeLayout',
2344
+ 'ViewGroup', 'ConstraintLayout', 'CoordinatorLayout',
2345
+ 'CardView', 'View', # 基础View也可能只是容器
2346
+ }
2347
+
2348
+ # 装饰类控件关键词(resource_id中包含这些关键词的通常可以过滤)
2349
+ # 支持匹配如 qylt_item_short_video_shadow_one 这样的命名
2350
+ DECORATIVE_KEYWORDS = {
2351
+ 'shadow', 'divider', 'separator', 'line', 'border',
2352
+ 'background', 'bg_', '_bg', 'decorative', 'decoration',
2353
+ '_shadow', 'shadow_', '_divider', 'divider_', '_line', 'line_'
2354
+ }
2355
+
2356
+ # Token 优化:构建精简元素(只返回非空字段)
2357
+ def build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name):
2358
+ """只返回有值的字段,节省 token"""
2359
+ item = {}
2360
+ if resource_id:
2361
+ # 精简 resource_id,只保留最后一段
2362
+ item['id'] = resource_id.split('/')[-1] if '/' in resource_id else resource_id
2363
+ if text:
2364
+ item['text'] = text
2365
+ if content_desc:
2366
+ item['desc'] = content_desc
2367
+ if bounds:
2368
+ item['bounds'] = bounds
2369
+ if likely_click:
2370
+ item['click'] = True # 启发式判断可点击
2371
+ # class 精简:只保留关键类型
2372
+ if class_name in ('EditText', 'TextInput', 'Button', 'ImageButton', 'CheckBox', 'Switch'):
2373
+ item['type'] = class_name
2374
+ return item
2375
+
1837
2376
  result = []
1838
2377
  for elem in elements:
1839
- if elem.get('clickable') or elem.get('focusable'):
1840
- result.append({
1841
- 'resource_id': elem.get('resource_id', ''),
1842
- 'text': elem.get('text', ''),
1843
- 'content_desc': elem.get('content_desc', ''),
1844
- 'bounds': elem.get('bounds', ''),
1845
- 'clickable': elem.get('clickable', False)
1846
- })
2378
+ # 获取元素属性
2379
+ class_name = elem.get('class_name', '')
2380
+ resource_id = elem.get('resource_id', '').strip()
2381
+ text = elem.get('text', '').strip()
2382
+ content_desc = elem.get('content_desc', '').strip()
2383
+ bounds = elem.get('bounds', '')
2384
+ clickable = elem.get('clickable', False)
2385
+ focusable = elem.get('focusable', False)
2386
+ scrollable = elem.get('scrollable', False)
2387
+ enabled = elem.get('enabled', True)
2388
+
2389
+ # 1. 过滤 bounds="[0,0][0,0]" 的视觉隐藏元素
2390
+ if bounds == '[0,0][0,0]':
2391
+ continue
2392
+
2393
+ # 2. 检查是否是功能控件(直接保留)
2394
+ if class_name in FUNCTIONAL_WIDGETS:
2395
+ # 使用启发式判断可点击性(替代不准确的 clickable 属性)
2396
+ likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
2397
+ item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
2398
+ if item:
2399
+ result.append(item)
2400
+ continue
2401
+
2402
+ # 3. 检查是否是容器控件
2403
+ if class_name in CONTAINER_WIDGETS:
2404
+ # 容器控件需要检查是否有业务相关的ID
2405
+ has_business_id = self._has_business_id(resource_id)
2406
+ if not has_business_id:
2407
+ # 无业务ID的容器控件,检查是否有其他有意义属性
2408
+ if not (clickable or focusable or scrollable or text or content_desc):
2409
+ # 所有属性都是默认值,过滤掉
2410
+ continue
2411
+ # 有业务ID或其他有意义属性,保留
2412
+ likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
2413
+ item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
2414
+ if item:
2415
+ result.append(item)
2416
+ continue
2417
+
2418
+ # 4. 检查是否是装饰类控件
2419
+ if resource_id:
2420
+ resource_id_lower = resource_id.lower()
2421
+ if any(keyword in resource_id_lower for keyword in DECORATIVE_KEYWORDS):
2422
+ # 是装饰类控件,且没有交互属性,过滤掉
2423
+ if not (clickable or focusable or text or content_desc):
2424
+ continue
2425
+
2426
+ # 5. 检查是否所有属性均为默认值
2427
+ if not (text or content_desc or resource_id or clickable or focusable or scrollable):
2428
+ # 所有属性都是默认值,过滤掉
2429
+ continue
2430
+
2431
+ # 6. 其他情况:有意义的元素保留
2432
+ likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
2433
+ item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
2434
+ if item:
2435
+ result.append(item)
2436
+
2437
+ # Token 优化:可选限制返回元素数量(默认不限制,确保准确度)
2438
+ if TOKEN_OPTIMIZATION and MAX_ELEMENTS > 0 and len(result) > MAX_ELEMENTS:
2439
+ # 仅在用户明确设置 MAX_ELEMENTS_RETURN 时才截断
2440
+ truncated = result[:MAX_ELEMENTS]
2441
+ truncated.append({
2442
+ '_truncated': True,
2443
+ '_total': len(result),
2444
+ '_shown': MAX_ELEMENTS
2445
+ })
2446
+ return truncated
2447
+
1847
2448
  return result
1848
2449
  except Exception as e:
1849
2450
  return [{"error": f"获取元素失败: {e}"}]
1850
2451
 
2452
+ def _get_page_texts(self, max_count: int = 15) -> List[str]:
2453
+ """获取页面关键文本列表(用于点击后快速确认页面变化)
2454
+
2455
+ Args:
2456
+ max_count: 最多返回的文本数量
2457
+
2458
+ Returns:
2459
+ 页面上的关键文本列表(去重)
2460
+ """
2461
+ try:
2462
+ if self._is_ios():
2463
+ ios_client = self._get_ios_client()
2464
+ if ios_client and hasattr(ios_client, 'wda'):
2465
+ # iOS: 获取所有 StaticText 的文本
2466
+ elements = ios_client.wda(type='XCUIElementTypeStaticText').find_elements()
2467
+ texts = set()
2468
+ for elem in elements[:50]: # 限制扫描数量
2469
+ try:
2470
+ name = elem.name or elem.label
2471
+ if name and len(name) > 1 and len(name) < 50:
2472
+ texts.add(name)
2473
+ except:
2474
+ pass
2475
+ return list(texts)[:max_count]
2476
+ return []
2477
+ else:
2478
+ # Android: 快速扫描 XML 获取文本
2479
+ xml_string = self.client.u2.dump_hierarchy(compressed=True)
2480
+ import xml.etree.ElementTree as ET
2481
+ root = ET.fromstring(xml_string)
2482
+
2483
+ texts = set()
2484
+ for elem in root.iter():
2485
+ text = elem.get('text', '').strip()
2486
+ desc = elem.get('content-desc', '').strip()
2487
+ # 只收集有意义的文本(长度2-30,非纯数字)
2488
+ for t in [text, desc]:
2489
+ if t and 2 <= len(t) <= 30 and not t.isdigit():
2490
+ texts.add(t)
2491
+ if len(texts) >= max_count * 2: # 收集足够后停止
2492
+ break
2493
+
2494
+ return list(texts)[:max_count]
2495
+ except Exception:
2496
+ return []
2497
+
2498
+ def _has_business_id(self, resource_id: str) -> bool:
2499
+ """
2500
+ 判断resource_id是否是业务相关的ID
2501
+
2502
+ 业务相关的ID通常包含:
2503
+ - 有意义的命名(不是自动生成的)
2504
+ - 不包含常见的自动生成模式
2505
+ """
2506
+ if not resource_id:
2507
+ return False
2508
+
2509
+ # 自动生成的ID模式(通常可以忽略)
2510
+ auto_generated_patterns = [
2511
+ r'^android:id/', # 系统ID
2512
+ r':id/\d+', # 数字ID
2513
+ r':id/view_\d+', # view_数字
2514
+ r':id/item_\d+', # item_数字
2515
+ ]
2516
+
2517
+ for pattern in auto_generated_patterns:
2518
+ if re.search(pattern, resource_id):
2519
+ return False
2520
+
2521
+ # 如果resource_id有实际内容且不是自动生成的,认为是业务ID
2522
+ # 排除一些常见的系统ID
2523
+ system_ids = ['android:id/content', 'android:id/statusBarBackground']
2524
+ if resource_id in system_ids:
2525
+ return False
2526
+
2527
+ return True
2528
+
2529
+ def _is_likely_clickable(self, class_name: str, resource_id: str, text: str,
2530
+ content_desc: str, clickable: bool, bounds: str) -> bool:
2531
+ """
2532
+ 启发式判断元素是否可能可点击
2533
+
2534
+ Android 的 clickable 属性经常不准确,因为:
2535
+ 1. 点击事件可能设置在父容器上
2536
+ 2. 使用 onTouchListener 而不是 onClick
2537
+ 3. RecyclerView item 通过 ItemClickListener 处理
2538
+
2539
+ 此方法通过多种规则推断元素的真实可点击性
2540
+ """
2541
+ # 规则1:clickable=true 肯定可点击
2542
+ if clickable:
2543
+ return True
2544
+
2545
+ # 规则2:特定类型的控件通常可点击
2546
+ TYPICALLY_CLICKABLE = {
2547
+ 'Button', 'ImageButton', 'CheckBox', 'RadioButton', 'Switch',
2548
+ 'ToggleButton', 'FloatingActionButton', 'Chip', 'TabView',
2549
+ 'EditText', 'TextInput', # 输入框可点击获取焦点
2550
+ }
2551
+ if class_name in TYPICALLY_CLICKABLE:
2552
+ return True
2553
+
2554
+ # 规则3:resource_id 包含可点击关键词
2555
+ if resource_id:
2556
+ id_lower = resource_id.lower()
2557
+ CLICK_KEYWORDS = [
2558
+ 'btn', 'button', 'click', 'tap', 'submit', 'confirm',
2559
+ 'cancel', 'close', 'back', 'next', 'prev', 'more',
2560
+ 'action', 'link', 'menu', 'tab', 'item', 'cell',
2561
+ 'card', 'avatar', 'icon', 'entry', 'option', 'arrow'
2562
+ ]
2563
+ for kw in CLICK_KEYWORDS:
2564
+ if kw in id_lower:
2565
+ return True
2566
+
2567
+ # 规则4:content_desc 包含可点击暗示
2568
+ if content_desc:
2569
+ desc_lower = content_desc.lower()
2570
+ CLICK_HINTS = ['点击', '按钮', '关闭', '返回', '更多', 'click', 'tap', 'button', 'close']
2571
+ for hint in CLICK_HINTS:
2572
+ if hint in desc_lower:
2573
+ return True
2574
+
2575
+ # 规则5:有 resource_id 或 content_desc 的小图标可能可点击
2576
+ # (纯 ImageView 不加判断,误判率太高)
2577
+ if class_name in ('ImageView', 'Image') and (resource_id or content_desc) and bounds:
2578
+ match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
2579
+ if match:
2580
+ x1, y1, x2, y2 = map(int, match.groups())
2581
+ w, h = x2 - x1, y2 - y1
2582
+ # 小图标(20-100px)更可能是按钮
2583
+ if 20 <= w <= 100 and 20 <= h <= 100:
2584
+ return True
2585
+
2586
+ # 规则6:移除(TextView 误判率太高,只依赖上面的规则)
2587
+ # 如果有 clickable=true 或 ID/desc 中有关键词,前面的规则已经覆盖
2588
+
2589
+ return False
2590
+
1851
2591
  def find_close_button(self) -> Dict:
1852
2592
  """智能查找关闭按钮(不点击,只返回位置)
1853
2593
 
@@ -1861,7 +2601,7 @@ class BasicMobileToolsLite:
1861
2601
  import re
1862
2602
 
1863
2603
  if self._is_ios():
1864
- return {"success": False, "message": "iOS 暂不支持,请使用截图+坐标点击"}
2604
+ return {"success": False, "msg": "iOS暂不支持"}
1865
2605
 
1866
2606
  # 获取屏幕尺寸
1867
2607
  screen_width = self.client.u2.info.get('displayWidth', 720)
@@ -1872,6 +2612,14 @@ class BasicMobileToolsLite:
1872
2612
  import xml.etree.ElementTree as ET
1873
2613
  root = ET.fromstring(xml_string)
1874
2614
 
2615
+ # 🔴 先检测是否有弹窗,避免误识别普通页面的按钮
2616
+ popup_bounds, popup_confidence = self._detect_popup_with_confidence(
2617
+ root, screen_width, screen_height
2618
+ )
2619
+
2620
+ if popup_bounds is None or popup_confidence < 0.5:
2621
+ return {"success": True, "popup": False}
2622
+
1875
2623
  # 关闭按钮特征
1876
2624
  close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
1877
2625
  candidates = []
@@ -1973,27 +2721,16 @@ class BasicMobileToolsLite:
1973
2721
  candidates.sort(key=lambda x: x['score'], reverse=True)
1974
2722
  best = candidates[0]
1975
2723
 
2724
+ # Token 优化:只返回最必要的信息
1976
2725
  return {
1977
2726
  "success": True,
1978
- "message": f"✅ 找到可能的关闭按钮",
1979
- "best_candidate": {
1980
- "reason": best['reason'],
1981
- "center": {"x": best['center_x'], "y": best['center_y']},
1982
- "percent": {"x": best['x_percent'], "y": best['y_percent']},
1983
- "bounds": best['bounds'],
1984
- "size": best['size'],
1985
- "score": best['score']
1986
- },
1987
- "click_command": f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})",
1988
- "other_candidates": [
1989
- {"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
1990
- for c in candidates[1:4]
1991
- ] if len(candidates) > 1 else [],
1992
- "screen_size": {"width": screen_width, "height": screen_height}
2727
+ "popup": True,
2728
+ "close": {"x": best['x_percent'], "y": best['y_percent']},
2729
+ "cmd": f"click_by_percent({best['x_percent']},{best['y_percent']})"
1993
2730
  }
1994
2731
 
1995
2732
  except Exception as e:
1996
- return {"success": False, "message": f"❌ 查找关闭按钮失败: {e}"}
2733
+ return {"success": False, "msg": str(e)}
1997
2734
 
1998
2735
  def close_popup(self) -> Dict:
1999
2736
  """智能关闭弹窗(改进版)
@@ -2016,7 +2753,7 @@ class BasicMobileToolsLite:
2016
2753
 
2017
2754
  # 获取屏幕尺寸
2018
2755
  if self._is_ios():
2019
- return {"success": False, "message": "iOS 暂不支持,请使用截图+坐标点击"}
2756
+ return {"success": False, "msg": "iOS暂不支持"}
2020
2757
 
2021
2758
  screen_width = self.client.u2.info.get('displayWidth', 720)
2022
2759
  screen_height = self.client.u2.info.get('displayHeight', 1280)
@@ -2036,53 +2773,18 @@ class BasicMobileToolsLite:
2036
2773
  root = ET.fromstring(xml_string)
2037
2774
  all_elements = list(root.iter())
2038
2775
 
2039
- # ===== 第一步:检测弹窗区域 =====
2040
- # 弹窗特征:非全屏、面积较大、通常在屏幕中央的容器
2041
- popup_containers = []
2042
- for idx, elem in enumerate(all_elements):
2043
- bounds_str = elem.attrib.get('bounds', '')
2044
- class_name = elem.attrib.get('class', '')
2045
-
2046
- if not bounds_str:
2047
- continue
2048
-
2049
- match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
2050
- if not match:
2051
- continue
2052
-
2053
- x1, y1, x2, y2 = map(int, match.groups())
2054
- width = x2 - x1
2055
- height = y2 - y1
2056
- area = width * height
2057
- screen_area = screen_width * screen_height
2058
-
2059
- # 弹窗容器特征:
2060
- # 1. 面积在屏幕的 10%-90% 之间(非全屏)
2061
- # 2. 宽度或高度不等于屏幕尺寸
2062
- # 3. 是容器类型(Layout/View/Dialog)
2063
- is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Container'])
2064
- area_ratio = area / screen_area
2065
- is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
2066
- is_reasonable_size = 0.08 < area_ratio < 0.9
2067
-
2068
- # 排除状态栏区域(y1 通常很小)
2069
- is_below_statusbar = y1 > 50
2070
-
2071
- if is_container and is_not_fullscreen and is_reasonable_size and is_below_statusbar:
2072
- popup_containers.append({
2073
- 'bounds': (x1, y1, x2, y2),
2074
- 'bounds_str': bounds_str,
2075
- 'area': area,
2076
- 'area_ratio': area_ratio,
2077
- 'idx': idx, # 元素在 XML 中的顺序(越后越上层)
2078
- 'class': class_name
2079
- })
2776
+ # ===== 第一步:使用严格的置信度检测弹窗区域 =====
2777
+ popup_bounds, popup_confidence = self._detect_popup_with_confidence(
2778
+ root, screen_width, screen_height
2779
+ )
2780
+
2781
+ # 如果置信度不够高,记录但继续尝试查找关闭按钮
2782
+ popup_detected = popup_bounds is not None and popup_confidence >= 0.6
2080
2783
 
2081
- # 选择最可能的弹窗容器(优先选择:XML 顺序靠后 + 面积适中)
2082
- if popup_containers:
2083
- # XML 顺序倒序(后出现的在上层),然后按面积适中程度排序
2084
- popup_containers.sort(key=lambda x: (x['idx'], -abs(x['area_ratio'] - 0.3)), reverse=True)
2085
- popup_bounds = popup_containers[0]['bounds']
2784
+ # 🔴 关键检查:如果没有检测到弹窗区域,直接返回"无弹窗"
2785
+ # 避免误点击普通页面上的"关闭"、"取消"等按钮
2786
+ if not popup_detected:
2787
+ return {"success": True, "popup": False}
2086
2788
 
2087
2789
  # ===== 第二步:在弹窗范围内查找关闭按钮 =====
2088
2790
  for idx, elem in enumerate(all_elements):
@@ -2214,73 +2916,16 @@ class BasicMobileToolsLite:
2214
2916
  'content_desc': content_desc,
2215
2917
  'x_percent': round(rel_x * 100, 1),
2216
2918
  'y_percent': round(rel_y * 100, 1),
2217
- 'in_popup': popup_bounds is not None
2919
+ 'in_popup': popup_detected
2218
2920
  })
2219
2921
 
2220
2922
  except ET.ParseError:
2221
2923
  pass
2222
2924
 
2223
2925
  if not close_candidates:
2224
- # 如果检测到弹窗区域,先尝试点击常见的关闭按钮位置
2225
- if popup_bounds:
2226
- px1, py1, px2, py2 = popup_bounds
2227
- popup_width = px2 - px1
2228
- popup_height = py2 - py1
2229
-
2230
- # 【优化】X按钮有三种常见位置:
2231
- # 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
2232
- # 2. 弹窗边界上方(浮动X按钮)
2233
- # 3. 弹窗正下方(底部关闭按钮)
2234
- offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
2235
- offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
2236
- offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
2237
-
2238
- try_positions = [
2239
- # 【最高优先级】弹窗内紧贴顶部边界
2240
- (px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
2241
- # 弹窗边界上方(浮动X按钮)
2242
- (px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
2243
- # 弹窗正下方中间(底部关闭按钮)
2244
- ((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
2245
- # 弹窗正上方中间
2246
- ((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
2247
- ]
2248
-
2249
- for try_x, try_y, position_name in try_positions:
2250
- if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
2251
- self.client.u2.click(try_x, try_y)
2252
- time.sleep(0.3)
2253
-
2254
- # 尝试后截图,让 AI 判断是否成功
2255
- screenshot_result = self.take_screenshot("尝试关闭后")
2256
- return {
2257
- "success": True,
2258
- "message": f"✅ 已尝试点击常见关闭按钮位置",
2259
- "tried_positions": [p[2] for p in try_positions],
2260
- "screenshot": screenshot_result.get("screenshot_path", ""),
2261
- "tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
2262
- }
2263
-
2264
- # 没有检测到弹窗区域,截图让 AI 分析
2265
- screenshot_result = self.take_screenshot(description="页面截图", compress=True)
2266
-
2267
- return {
2268
- "success": False,
2269
- "message": "❌ 未检测到弹窗区域,已截图供 AI 分析",
2270
- "action_required": "请查看截图找到关闭按钮,调用 mobile_click_at_coords 点击",
2271
- "screenshot": screenshot_result.get("screenshot_path", ""),
2272
- "screen_size": {"width": screen_width, "height": screen_height},
2273
- "image_size": {
2274
- "width": screenshot_result.get("image_width", screen_width),
2275
- "height": screenshot_result.get("image_height", screen_height)
2276
- },
2277
- "original_size": {
2278
- "width": screenshot_result.get("original_img_width", screen_width),
2279
- "height": screenshot_result.get("original_img_height", screen_height)
2280
- },
2281
- "search_areas": ["弹窗右上角", "弹窗正上方", "弹窗下方中间", "屏幕右上角"],
2282
- "time_warning": "⚠️ 截图分析期间弹窗可能自动消失。如果是定时弹窗,建议等待其自动消失。"
2283
- }
2926
+ if popup_detected and popup_bounds:
2927
+ return {"success": False, "fallback": "vision", "popup": True}
2928
+ return {"success": True, "popup": False}
2284
2929
 
2285
2930
  # 按得分排序,取最可能的
2286
2931
  close_candidates.sort(key=lambda x: x['score'], reverse=True)
@@ -2290,49 +2935,30 @@ class BasicMobileToolsLite:
2290
2935
  self.client.u2.click(best['center_x'], best['center_y'])
2291
2936
  time.sleep(0.5)
2292
2937
 
2293
- # 点击后截图,让 AI 判断是否成功
2294
- screenshot_result = self.take_screenshot("关闭弹窗后")
2295
-
2296
- # 记录操作(使用百分比,跨设备兼容)
2297
- self._record_operation(
2298
- 'click',
2299
- x=best['center_x'],
2300
- y=best['center_y'],
2301
- x_percent=best['x_percent'],
2302
- y_percent=best['y_percent'],
2303
- screen_width=screen_width,
2304
- screen_height=screen_height,
2305
- ref=f"close_popup_{best['position']}"
2306
- )
2938
+ # 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
2939
+ app_check = self._check_app_switched()
2940
+ return_result = None
2307
2941
 
2308
- # 返回候选按钮列表,让 AI 看截图判断
2309
- # 如果弹窗还在,AI 可以选择点击其他候选按钮
2310
- return {
2311
- "success": True,
2312
- "message": f"✅ 已点击关闭按钮 ({best['position']}): ({best['center_x']}, {best['center_y']})",
2313
- "clicked": {
2314
- "position": best['position'],
2315
- "match_type": best['match_type'],
2316
- "coords": (best['center_x'], best['center_y']),
2317
- "percent": (best['x_percent'], best['y_percent'])
2318
- },
2319
- "screenshot": screenshot_result.get("screenshot_path", ""),
2320
- "popup_detected": popup_bounds is not None,
2321
- "popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
2322
- "other_candidates": [
2323
- {
2324
- "position": c['position'],
2325
- "type": c['match_type'],
2326
- "coords": (c['center_x'], c['center_y']),
2327
- "percent": (c['x_percent'], c['y_percent'])
2328
- }
2329
- for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
2330
- ],
2331
- "tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置;如果误点跳转了,请按返回键"
2332
- }
2942
+ if app_check['switched']:
2943
+ # 应用已跳转,说明弹窗去除失败,尝试返回目标应用
2944
+ return_result = self._return_to_target_app()
2945
+
2946
+ # 记录操作
2947
+ self._record_click('percent', f"{best['x_percent']}%,{best['y_percent']}%",
2948
+ best['x_percent'], best['y_percent'],
2949
+ element_desc=f"关闭按钮({best['position']})")
2950
+
2951
+ # Token 优化:精简返回值
2952
+ result = {"success": True, "clicked": True}
2953
+ if app_check['switched']:
2954
+ result["switched"] = True
2955
+ if return_result:
2956
+ result["returned"] = return_result['success']
2957
+
2958
+ return result
2333
2959
 
2334
2960
  except Exception as e:
2335
- return {"success": False, "message": f"❌ 关闭弹窗失败: {e}"}
2961
+ return {"success": False, "msg": str(e)}
2336
2962
 
2337
2963
  def _get_position_name(self, rel_x: float, rel_y: float) -> str:
2338
2964
  """根据相对坐标获取位置名称"""
@@ -2375,6 +3001,308 @@ class BasicMobileToolsLite:
2375
3001
  return 0.8
2376
3002
  else: # 中间区域
2377
3003
  return 0.5
3004
+
3005
+ def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) -> tuple:
3006
+ """严格的弹窗检测 - 使用置信度评分,避免误识别普通页面
3007
+
3008
+ 真正的弹窗特征:
3009
+ 1. class 名称包含 Dialog/Popup/Alert/Modal/BottomSheet(强特征)
3010
+ 2. resource-id 包含 dialog/popup/alert/modal(强特征)
3011
+ 3. 有遮罩层(大面积半透明 View 在弹窗之前)
3012
+ 4. 居中显示且非全屏
3013
+ 5. XML 层级靠后且包含可交互元素
3014
+
3015
+ Returns:
3016
+ (popup_bounds, confidence) 或 (None, 0)
3017
+ confidence >= 0.6 才认为是弹窗
3018
+ """
3019
+ import re
3020
+
3021
+ screen_area = screen_width * screen_height
3022
+
3023
+ # 收集所有元素信息
3024
+ all_elements = []
3025
+ for idx, elem in enumerate(root.iter()):
3026
+ bounds_str = elem.attrib.get('bounds', '')
3027
+ if not bounds_str:
3028
+ continue
3029
+
3030
+ match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
3031
+ if not match:
3032
+ continue
3033
+
3034
+ x1, y1, x2, y2 = map(int, match.groups())
3035
+ width = x2 - x1
3036
+ height = y2 - y1
3037
+ area = width * height
3038
+
3039
+ class_name = elem.attrib.get('class', '')
3040
+ resource_id = elem.attrib.get('resource-id', '')
3041
+ clickable = elem.attrib.get('clickable', 'false') == 'true'
3042
+
3043
+ all_elements.append({
3044
+ 'idx': idx,
3045
+ 'bounds': (x1, y1, x2, y2),
3046
+ 'width': width,
3047
+ 'height': height,
3048
+ 'area': area,
3049
+ 'area_ratio': area / screen_area if screen_area > 0 else 0,
3050
+ 'class': class_name,
3051
+ 'resource_id': resource_id,
3052
+ 'clickable': clickable,
3053
+ 'center_x': (x1 + x2) // 2,
3054
+ 'center_y': (y1 + y2) // 2,
3055
+ })
3056
+
3057
+ if not all_elements:
3058
+ return None, 0
3059
+
3060
+ # 弹窗检测关键词
3061
+ dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
3062
+ dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
3063
+
3064
+ popup_candidates = []
3065
+ has_mask_layer = False
3066
+ mask_idx = -1
3067
+
3068
+ for elem in all_elements:
3069
+ x1, y1, x2, y2 = elem['bounds']
3070
+ class_name = elem['class']
3071
+ resource_id = elem['resource_id']
3072
+ area_ratio = elem['area_ratio']
3073
+
3074
+ # 检测遮罩层(大面积、几乎全屏、通常是 FrameLayout/View)
3075
+ if area_ratio > 0.85 and elem['width'] >= screen_width * 0.95:
3076
+ # 可能是遮罩层,记录位置
3077
+ if 'FrameLayout' in class_name or 'View' in class_name:
3078
+ has_mask_layer = True
3079
+ mask_idx = elem['idx']
3080
+
3081
+ # 跳过全屏元素
3082
+ if area_ratio > 0.9:
3083
+ continue
3084
+
3085
+ # 跳过太小的元素
3086
+ if area_ratio < 0.05:
3087
+ continue
3088
+
3089
+ # 跳过状态栏区域
3090
+ if y1 < 50:
3091
+ continue
3092
+
3093
+ confidence = 0.0
3094
+
3095
+ # 【强特征】class 名称包含弹窗关键词 (+0.5)
3096
+ if any(kw in class_name for kw in dialog_class_keywords):
3097
+ confidence += 0.5
3098
+
3099
+ # 【强特征】resource-id 包含弹窗关键词 (+0.4)
3100
+ if any(kw in resource_id.lower() for kw in dialog_id_keywords):
3101
+ confidence += 0.4
3102
+
3103
+ # 【中等特征】居中显示 (+0.2)
3104
+ center_x = elem['center_x']
3105
+ center_y = elem['center_y']
3106
+ is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
3107
+ is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
3108
+ if is_centered_x and is_centered_y:
3109
+ confidence += 0.2
3110
+ elif is_centered_x:
3111
+ confidence += 0.1
3112
+
3113
+ # 【中等特征】非全屏但有一定大小 (+0.15)
3114
+ if 0.15 < area_ratio < 0.75:
3115
+ confidence += 0.15
3116
+
3117
+ # 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
3118
+ if elem['idx'] > len(all_elements) * 0.5:
3119
+ confidence += 0.1
3120
+
3121
+ # 【弱特征】有遮罩层且在遮罩层之后 (+0.15)
3122
+ if has_mask_layer and elem['idx'] > mask_idx:
3123
+ confidence += 0.15
3124
+
3125
+ # 只有达到阈值才加入候选
3126
+ if confidence >= 0.3:
3127
+ popup_candidates.append({
3128
+ 'bounds': elem['bounds'],
3129
+ 'confidence': confidence,
3130
+ 'class': class_name,
3131
+ 'resource_id': resource_id,
3132
+ 'idx': elem['idx']
3133
+ })
3134
+
3135
+ if not popup_candidates:
3136
+ return None, 0
3137
+
3138
+ # 选择置信度最高的
3139
+ popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
3140
+ best = popup_candidates[0]
3141
+
3142
+ # 只有置信度 >= 0.6 才返回弹窗
3143
+ if best['confidence'] >= 0.6:
3144
+ return best['bounds'], best['confidence']
3145
+
3146
+ return None, best['confidence']
3147
+
3148
+ def start_toast_watch(self) -> Dict:
3149
+ """开始监听 Toast(仅 Android)
3150
+
3151
+ ⚠️ 必须在执行操作之前调用!
3152
+
3153
+ 正确流程:
3154
+ 1. 调用 mobile_start_toast_watch() 开始监听
3155
+ 2. 执行操作(如点击提交按钮)
3156
+ 3. 调用 mobile_get_toast() 获取 Toast 内容
3157
+
3158
+ Returns:
3159
+ 监听状态
3160
+ """
3161
+ if self._is_ios():
3162
+ return {
3163
+ "success": False,
3164
+ "message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
3165
+ }
3166
+
3167
+ try:
3168
+ # 清除缓存并开始监听
3169
+ self.client.u2.toast.reset()
3170
+ return {
3171
+ "success": True,
3172
+ "message": "✅ Toast 监听已开启,请立即执行操作,然后调用 mobile_get_toast 获取结果"
3173
+ }
3174
+ except Exception as e:
3175
+ return {
3176
+ "success": False,
3177
+ "message": f"❌ 开启 Toast 监听失败: {e}"
3178
+ }
3179
+
3180
+ def get_toast(self, timeout: float = 5.0, reset_first: bool = False) -> Dict:
3181
+ """获取 Toast 消息(仅 Android)
3182
+
3183
+ Toast 是 Android 系统级的短暂提示消息,常用于显示操作结果。
3184
+
3185
+ ⚠️ 推荐用法(两步走):
3186
+ 1. 先调用 mobile_start_toast_watch() 开始监听
3187
+ 2. 执行操作(如点击提交按钮)
3188
+ 3. 调用 mobile_get_toast() 获取 Toast
3189
+
3190
+ 或者设置 reset_first=True,会自动 reset 后等待(适合操作已自动触发的场景)
3191
+
3192
+ Args:
3193
+ timeout: 等待 Toast 出现的超时时间(秒),默认 5 秒
3194
+ reset_first: 是否先 reset(清除旧缓存),默认 False
3195
+
3196
+ Returns:
3197
+ 包含 Toast 消息的字典
3198
+ """
3199
+ if self._is_ios():
3200
+ return {
3201
+ "success": False,
3202
+ "message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
3203
+ }
3204
+
3205
+ try:
3206
+ if reset_first:
3207
+ # 清除旧缓存,适合等待即将出现的 Toast
3208
+ self.client.u2.toast.reset()
3209
+
3210
+ # 等待并获取 Toast 消息
3211
+ toast_message = self.client.u2.toast.get_message(
3212
+ wait_timeout=timeout,
3213
+ default=None
3214
+ )
3215
+
3216
+ if toast_message:
3217
+ return {
3218
+ "success": True,
3219
+ "toast_found": True,
3220
+ "message": toast_message,
3221
+ "tip": "Toast 消息获取成功"
3222
+ }
3223
+ else:
3224
+ return {
3225
+ "success": True,
3226
+ "toast_found": False,
3227
+ "message": None,
3228
+ "tip": f"在 {timeout} 秒内未检测到 Toast。提示:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具"
3229
+ }
3230
+ except Exception as e:
3231
+ return {
3232
+ "success": False,
3233
+ "message": f"❌ 获取 Toast 失败: {e}"
3234
+ }
3235
+
3236
+ def assert_toast(self, expected_text: str, timeout: float = 5.0, contains: bool = True) -> Dict:
3237
+ """断言 Toast 消息(仅 Android)
3238
+
3239
+ 等待 Toast 出现并验证内容是否符合预期。
3240
+
3241
+ ⚠️ 推荐用法:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具
3242
+
3243
+ Args:
3244
+ expected_text: 期望的 Toast 文本
3245
+ timeout: 等待超时时间(秒)
3246
+ contains: True 表示包含匹配,False 表示精确匹配
3247
+
3248
+ Returns:
3249
+ 断言结果
3250
+ """
3251
+ if self._is_ios():
3252
+ return {
3253
+ "success": False,
3254
+ "passed": False,
3255
+ "message": "❌ iOS 不支持 Toast 检测"
3256
+ }
3257
+
3258
+ try:
3259
+ # 获取 Toast(不 reset,假设之前已经调用过 start_toast_watch)
3260
+ toast_message = self.client.u2.toast.get_message(
3261
+ wait_timeout=timeout,
3262
+ default=None
3263
+ )
3264
+
3265
+ if toast_message is None:
3266
+ return {
3267
+ "success": True,
3268
+ "passed": False,
3269
+ "expected": expected_text,
3270
+ "actual": None,
3271
+ "message": f"❌ 断言失败:未检测到 Toast 消息"
3272
+ }
3273
+
3274
+ # 匹配检查
3275
+ if contains:
3276
+ passed = expected_text in toast_message
3277
+ match_type = "包含"
3278
+ else:
3279
+ passed = expected_text == toast_message
3280
+ match_type = "精确"
3281
+
3282
+ if passed:
3283
+ return {
3284
+ "success": True,
3285
+ "passed": True,
3286
+ "expected": expected_text,
3287
+ "actual": toast_message,
3288
+ "match_type": match_type,
3289
+ "message": f"✅ Toast 断言通过:'{toast_message}'"
3290
+ }
3291
+ else:
3292
+ return {
3293
+ "success": True,
3294
+ "passed": False,
3295
+ "expected": expected_text,
3296
+ "actual": toast_message,
3297
+ "match_type": match_type,
3298
+ "message": f"❌ Toast 断言失败:期望 '{expected_text}',实际 '{toast_message}'"
3299
+ }
3300
+ except Exception as e:
3301
+ return {
3302
+ "success": False,
3303
+ "passed": False,
3304
+ "message": f"❌ Toast 断言异常: {e}"
3305
+ }
2378
3306
 
2379
3307
  def assert_text(self, text: str) -> Dict:
2380
3308
  """检查页面是否包含文本(支持精确匹配和包含匹配)"""
@@ -2460,11 +3388,16 @@ class BasicMobileToolsLite:
2460
3388
  f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
2461
3389
  "",
2462
3390
  "定位策略(按优先级):",
2463
- "1. ID 定位 - 最稳定,跨设备兼容",
2464
- "2. 文本定位 - 稳定,跨设备兼容",
3391
+ "1. 文本定位 - 最稳定,跨设备兼容",
3392
+ "2. ID 定位 - 稳定,跨设备兼容",
2465
3393
  "3. 百分比定位 - 跨分辨率兼容(坐标自动转换)",
3394
+ "",
3395
+ "运行方式:",
3396
+ " pytest {filename} -v # 使用 pytest 运行",
3397
+ " python {filename} # 直接运行",
2466
3398
  f'"""',
2467
3399
  "import time",
3400
+ "import pytest",
2468
3401
  "import uiautomator2 as u2",
2469
3402
  "",
2470
3403
  f'PACKAGE_NAME = "{package_name}"',
@@ -2540,22 +3473,52 @@ class BasicMobileToolsLite:
2540
3473
  " return True",
2541
3474
  "",
2542
3475
  "",
2543
- "def test_main():",
2544
- " # 连接设备",
2545
- " d = u2.connect()",
2546
- " d.implicitly_wait(10) # 设置全局等待",
2547
- " ",
2548
- " # 启动应用",
2549
- f" d.app_start(PACKAGE_NAME)",
2550
- " time.sleep(LAUNCH_WAIT) # 等待启动(可调整)",
3476
+ "def swipe_direction(d, direction):",
3477
+ ' """',
3478
+ ' 通用滑动方法(兼容所有 uiautomator2 版本)',
3479
+ ' ',
3480
+ ' Args:',
3481
+ ' d: uiautomator2 设备对象',
3482
+ ' direction: 滑动方向 (up/down/left/right)',
3483
+ ' """',
3484
+ " info = d.info",
3485
+ " width = info.get('displayWidth', 0)",
3486
+ " height = info.get('displayHeight', 0)",
3487
+ " cx, cy = width // 2, height // 2",
2551
3488
  " ",
2552
- " # 尝试关闭启动广告(可选,根据 App 情况调整)",
3489
+ " if direction == 'up':",
3490
+ " d.swipe(cx, int(height * 0.8), cx, int(height * 0.3))",
3491
+ " elif direction == 'down':",
3492
+ " d.swipe(cx, int(height * 0.3), cx, int(height * 0.8))",
3493
+ " elif direction == 'left':",
3494
+ " d.swipe(int(width * 0.8), cy, int(width * 0.2), cy)",
3495
+ " elif direction == 'right':",
3496
+ " d.swipe(int(width * 0.2), cy, int(width * 0.8), cy)",
3497
+ " return True",
3498
+ "",
3499
+ "",
3500
+ "# ========== pytest fixture ==========",
3501
+ "@pytest.fixture(scope='function')",
3502
+ "def device():",
3503
+ ' """pytest fixture: 连接设备并启动应用"""',
3504
+ " d = u2.connect()",
3505
+ " d.implicitly_wait(10)",
3506
+ " d.app_start(PACKAGE_NAME)",
3507
+ " time.sleep(LAUNCH_WAIT)",
2553
3508
  " if CLOSE_AD_ON_LAUNCH:",
2554
3509
  " close_ad_if_exists(d)",
3510
+ " yield d",
3511
+ " # 测试结束后可选择关闭应用",
3512
+ " # d.app_stop(PACKAGE_NAME)",
3513
+ "",
3514
+ "",
3515
+ f"def test_{safe_name}(device):",
3516
+ ' """测试用例主函数"""',
3517
+ " d = device",
2555
3518
  " ",
2556
3519
  ]
2557
3520
 
2558
- # 生成操作代码(跳过启动应用相关操作,因为脚本头部已处理)
3521
+ # 生成操作代码(使用标准记录格式,逻辑更简洁)
2559
3522
  step_num = 0
2560
3523
  for op in self.operation_history:
2561
3524
  action = op.get('action')
@@ -2567,130 +3530,122 @@ class BasicMobileToolsLite:
2567
3530
  step_num += 1
2568
3531
 
2569
3532
  if action == 'click':
2570
- ref = op.get('ref', '')
2571
- element = op.get('element', '')
2572
- has_coords = 'x' in op and 'y' in op
2573
- has_percent = 'x_percent' in op and 'y_percent' in op
2574
-
2575
- # 判断 ref 是否为坐标格式(coords_ 或 coords:)
2576
- is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
2577
- is_percent_ref = ref.startswith('percent_')
2578
-
2579
- # 优先级:ID > 文本 > 百分比 > 坐标(兜底)
2580
- if ref and (':id/' in ref or ref.startswith('com.')):
2581
- # 1️⃣ 使用 resource-id(最稳定)
2582
- script_lines.append(f" # 步骤{step_num}: 点击元素 (ID定位,最稳定)")
2583
- script_lines.append(f" safe_click(d, d(resourceId='{ref}'))")
2584
- elif ref and not is_coords_ref and not is_percent_ref and ':' not in ref:
2585
- # 2️⃣ 使用文本(稳定)- 排除 "text:xxx" 等带冒号的格式
2586
- script_lines.append(f" # 步骤{step_num}: 点击文本 '{ref}' (文本定位)")
2587
- script_lines.append(f" safe_click(d, d(text='{ref}'))")
2588
- elif ref and ':' in ref and not is_coords_ref and not is_percent_ref:
2589
- # 2️⃣-b 使用文本(Android 的 text:xxx 或 description:xxx 格式)
2590
- # 提取冒号后面的实际文本值
2591
- actual_text = ref.split(':', 1)[1] if ':' in ref else ref
2592
- script_lines.append(f" # 步骤{step_num}: 点击文本 '{actual_text}' (文本定位)")
2593
- script_lines.append(f" safe_click(d, d(text='{actual_text}'))")
2594
- elif has_percent:
2595
- # 3️⃣ 使用百分比(跨分辨率兼容)
2596
- x_pct = op['x_percent']
2597
- y_pct = op['y_percent']
2598
- desc = f" ({element})" if element else ""
2599
- script_lines.append(f" # 步骤{step_num}: 点击位置{desc} (百分比定位,跨分辨率兼容)")
2600
- script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
2601
- elif has_coords:
2602
- # 4️⃣ 坐标兜底(不推荐,仅用于无法获取百分比的情况)
2603
- desc = f" ({element})" if element else ""
2604
- script_lines.append(f" # 步骤{step_num}: 点击坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
2605
- script_lines.append(f" d.click({op['x']}, {op['y']})")
3533
+ # 新格式:使用 locator_type 和 locator_value
3534
+ locator_type = op.get('locator_type', '')
3535
+ locator_value = op.get('locator_value', '')
3536
+ locator_attr = op.get('locator_attr', 'text')
3537
+ element_desc = op.get('element_desc', '')
3538
+ x_pct = op.get('x_percent', 0)
3539
+ y_pct = op.get('y_percent', 0)
3540
+
3541
+ # 转义单引号
3542
+ value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
3543
+
3544
+ if locator_type == 'text':
3545
+ # 文本定位(最稳定)
3546
+ script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (文本定位)")
3547
+ if locator_attr == 'description':
3548
+ script_lines.append(f" safe_click(d, d(description='{value_escaped}'))")
3549
+ elif locator_attr == 'descriptionContains':
3550
+ script_lines.append(f" safe_click(d, d(descriptionContains='{value_escaped}'))")
3551
+ elif locator_attr == 'textContains':
3552
+ script_lines.append(f" safe_click(d, d(textContains='{value_escaped}'))")
3553
+ else:
3554
+ script_lines.append(f" safe_click(d, d(text='{value_escaped}'))")
3555
+ elif locator_type == 'id':
3556
+ # ID 定位(稳定)
3557
+ script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (ID定位)")
3558
+ script_lines.append(f" safe_click(d, d(resourceId='{value_escaped}'))")
3559
+ elif locator_type == 'percent':
3560
+ # 百分比定位(跨分辨率兼容)
3561
+ script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (百分比定位)")
3562
+ script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
2606
3563
  else:
2607
- continue # 无效操作,跳过
2608
-
2609
- script_lines.append(" time.sleep(0.5) # 等待响应")
3564
+ # 兼容旧格式
3565
+ ref = op.get('ref', '')
3566
+ if ref:
3567
+ ref_escaped = ref.replace("'", "\\'")
3568
+ script_lines.append(f" # 步骤{step_num}: 点击 '{ref}'")
3569
+ script_lines.append(f" safe_click(d, d(text='{ref_escaped}'))")
3570
+ else:
3571
+ continue
3572
+
3573
+ script_lines.append(" time.sleep(0.5)")
2610
3574
  script_lines.append(" ")
2611
3575
 
2612
3576
  elif action == 'input':
2613
3577
  text = op.get('text', '')
2614
- ref = op.get('ref', '')
2615
- has_coords = 'x' in op and 'y' in op
2616
- has_percent = 'x_percent' in op and 'y_percent' in op
2617
-
2618
- # 判断 ref 是否为坐标格式
2619
- is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
2620
-
2621
- # 优先使用 ID,其次百分比,最后坐标
2622
- if ref and not is_coords_ref and (':id/' in ref or ref.startswith('com.')):
2623
- # 完整格式的 resource-id
2624
- script_lines.append(f" # 步骤{step_num}: 输入文本 '{text}' (ID定位)")
2625
- script_lines.append(f" d(resourceId='{ref}').set_text('{text}')")
2626
- elif ref and not is_coords_ref and not has_coords:
2627
- # 简短格式的 resource-id(不包含 com. 或 :id/)
2628
- script_lines.append(f" # 步骤{step_num}: 输入文本 '{text}' (ID定位)")
2629
- script_lines.append(f" d(resourceId='{ref}').set_text('{text}')")
2630
- elif has_percent:
2631
- x_pct = op['x_percent']
2632
- y_pct = op['y_percent']
2633
- script_lines.append(f" # 步骤{step_num}: 点击后输入 (百分比定位)")
3578
+ locator_type = op.get('locator_type', '')
3579
+ locator_value = op.get('locator_value', '')
3580
+ x_pct = op.get('x_percent', 0)
3581
+ y_pct = op.get('y_percent', 0)
3582
+
3583
+ text_escaped = text.replace("'", "\\'")
3584
+ value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
3585
+
3586
+ if locator_type == 'id':
3587
+ script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (ID定位)")
3588
+ script_lines.append(f" d(resourceId='{value_escaped}').set_text('{text_escaped}')")
3589
+ elif locator_type == 'class':
3590
+ script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (类名定位)")
3591
+ script_lines.append(f" d(className='android.widget.EditText').set_text('{text_escaped}')")
3592
+ elif x_pct > 0 and y_pct > 0:
3593
+ script_lines.append(f" # 步骤{step_num}: 点击后输入 '{text}'")
2634
3594
  script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
2635
- script_lines.append(f" time.sleep(0.3)")
2636
- script_lines.append(f" d.send_keys('{text}')")
2637
- elif has_coords:
2638
- script_lines.append(f" # 步骤{step_num}: 点击坐标后输入 (⚠️ 可能不兼容其他分辨率)")
2639
- script_lines.append(f" d.click({op['x']}, {op['y']})")
2640
- script_lines.append(f" time.sleep(0.3)")
2641
- script_lines.append(f" d.send_keys('{text}')")
3595
+ script_lines.append(" time.sleep(0.3)")
3596
+ script_lines.append(f" d.send_keys('{text_escaped}')")
2642
3597
  else:
2643
- # 兜底:无法识别的格式,跳过
2644
- continue
3598
+ # 兼容旧格式
3599
+ ref = op.get('ref', '')
3600
+ if ref:
3601
+ script_lines.append(f" # 步骤{step_num}: 输入 '{text}'")
3602
+ script_lines.append(f" d(resourceId='{ref}').set_text('{text_escaped}')")
3603
+ else:
3604
+ continue
3605
+
2645
3606
  script_lines.append(" time.sleep(0.5)")
2646
3607
  script_lines.append(" ")
2647
3608
 
2648
3609
  elif action == 'long_press':
2649
- ref = op.get('ref', '')
2650
- element = op.get('element', '')
3610
+ locator_type = op.get('locator_type', '')
3611
+ locator_value = op.get('locator_value', '')
3612
+ locator_attr = op.get('locator_attr', 'text')
3613
+ element_desc = op.get('element_desc', '')
2651
3614
  duration = op.get('duration', 1.0)
2652
- has_coords = 'x' in op and 'y' in op
2653
- has_percent = 'x_percent' in op and 'y_percent' in op
2654
-
2655
- # 判断 ref 是否为坐标格式
2656
- is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
2657
- is_percent_ref = ref.startswith('percent_')
2658
-
2659
- # 优先级:ID > 文本 > 百分比 > 坐标
2660
- if ref and (':id/' in ref or ref.startswith('com.')):
2661
- # 使用 resource-id
2662
- script_lines.append(f" # 步骤{step_num}: 长按元素 (ID定位,最稳定)")
2663
- script_lines.append(f" d(resourceId='{ref}').long_click(duration={duration})")
2664
- elif ref and not is_coords_ref and not is_percent_ref and ':' not in ref:
2665
- # 使用文本
2666
- script_lines.append(f" # 步骤{step_num}: 长按文本 '{ref}' (文本定位)")
2667
- script_lines.append(f" d(text='{ref}').long_click(duration={duration})")
2668
- elif ref and ':' in ref and not is_coords_ref and not is_percent_ref:
2669
- actual_text = ref.split(':', 1)[1] if ':' in ref else ref
2670
- script_lines.append(f" # 步骤{step_num}: 长按文本 '{actual_text}' (文本定位)")
2671
- script_lines.append(f" d(text='{actual_text}').long_click(duration={duration})")
2672
- elif has_percent:
2673
- # 使用百分比
2674
- x_pct = op['x_percent']
2675
- y_pct = op['y_percent']
2676
- desc = f" ({element})" if element else ""
2677
- script_lines.append(f" # 步骤{step_num}: 长按位置{desc} (百分比定位,跨分辨率兼容)")
2678
- script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
2679
- elif has_coords:
2680
- # 坐标兜底
2681
- desc = f" ({element})" if element else ""
2682
- script_lines.append(f" # 步骤{step_num}: 长按坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
2683
- script_lines.append(f" d.long_click({op['x']}, {op['y']}, duration={duration})")
3615
+ x_pct = op.get('x_percent', 0)
3616
+ y_pct = op.get('y_percent', 0)
3617
+
3618
+ value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
3619
+
3620
+ if locator_type == 'text':
3621
+ script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
3622
+ if locator_attr == 'description':
3623
+ script_lines.append(f" d(description='{value_escaped}').long_click(duration={duration})")
3624
+ else:
3625
+ script_lines.append(f" d(text='{value_escaped}').long_click(duration={duration})")
3626
+ elif locator_type == 'id':
3627
+ script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
3628
+ script_lines.append(f" d(resourceId='{value_escaped}').long_click(duration={duration})")
3629
+ elif locator_type == 'percent':
3630
+ script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
3631
+ script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration})")
2684
3632
  else:
2685
- continue
2686
-
2687
- script_lines.append(" time.sleep(0.5) # 等待响应")
3633
+ # 兼容旧格式
3634
+ ref = op.get('ref', '')
3635
+ if ref:
3636
+ ref_escaped = ref.replace("'", "\\'")
3637
+ script_lines.append(f" # 步骤{step_num}: 长按 '{ref}'")
3638
+ script_lines.append(f" d(text='{ref_escaped}').long_click(duration={duration})")
3639
+ else:
3640
+ continue
3641
+
3642
+ script_lines.append(" time.sleep(0.5)")
2688
3643
  script_lines.append(" ")
2689
3644
 
2690
3645
  elif action == 'swipe':
2691
3646
  direction = op.get('direction', 'up')
2692
3647
  script_lines.append(f" # 步骤{step_num}: 滑动 {direction}")
2693
- script_lines.append(f" d.swipe_ext('{direction}')")
3648
+ script_lines.append(f" swipe_direction(d, '{direction}')")
2694
3649
  script_lines.append(" time.sleep(0.5)")
2695
3650
  script_lines.append(" ")
2696
3651
 
@@ -2705,8 +3660,16 @@ class BasicMobileToolsLite:
2705
3660
  " print('✅ 测试完成')",
2706
3661
  "",
2707
3662
  "",
3663
+ "# ========== 直接运行入口 ==========",
2708
3664
  "if __name__ == '__main__':",
2709
- " test_main()",
3665
+ " # 直接运行时,手动创建设备连接",
3666
+ " _d = u2.connect()",
3667
+ " _d.implicitly_wait(10)",
3668
+ " _d.app_start(PACKAGE_NAME)",
3669
+ " time.sleep(LAUNCH_WAIT)",
3670
+ " if CLOSE_AD_ON_LAUNCH:",
3671
+ " close_ad_if_exists(_d)",
3672
+ f" test_{safe_name}(_d)",
2710
3673
  ])
2711
3674
 
2712
3675
  script = '\n'.join(script_lines)
@@ -2715,8 +3678,11 @@ class BasicMobileToolsLite:
2715
3678
  output_dir = Path("tests")
2716
3679
  output_dir.mkdir(exist_ok=True)
2717
3680
 
3681
+ # 确保文件名符合 pytest 规范(以 test_ 开头)
2718
3682
  if not filename.endswith('.py'):
2719
3683
  filename = f"{filename}.py"
3684
+ if not filename.startswith('test_'):
3685
+ filename = f"test_{filename}"
2720
3686
 
2721
3687
  file_path = output_dir / filename
2722
3688
  file_path.write_text(script, encoding='utf-8')
@@ -2724,7 +3690,7 @@ class BasicMobileToolsLite:
2724
3690
  return {
2725
3691
  "success": True,
2726
3692
  "file_path": str(file_path),
2727
- "message": f"✅ 脚本已生成: {file_path}",
3693
+ "message": f"✅ 脚本已生成: {file_path}\n💡 运行方式: pytest {file_path} -v 或 python {file_path}",
2728
3694
  "operations_count": len(self.operation_history),
2729
3695
  "preview": script[:500] + "..."
2730
3696
  }
@@ -2893,10 +3859,28 @@ class BasicMobileToolsLite:
2893
3859
  try:
2894
3860
  import xml.etree.ElementTree as ET
2895
3861
 
2896
- # ========== 第1步:控件树查找关闭按钮 ==========
3862
+ # ========== 第0步:先检测是否有弹窗 ==========
2897
3863
  xml_string = self.client.u2.dump_hierarchy(compressed=False)
2898
3864
  root = ET.fromstring(xml_string)
2899
3865
 
3866
+ screen_width = self.client.u2.info.get('displayWidth', 1440)
3867
+ screen_height = self.client.u2.info.get('displayHeight', 3200)
3868
+
3869
+ popup_bounds, popup_confidence = self._detect_popup_with_confidence(
3870
+ root, screen_width, screen_height
3871
+ )
3872
+
3873
+ # 如果没有检测到弹窗,直接返回"无弹窗"
3874
+ if popup_bounds is None or popup_confidence < 0.5:
3875
+ result["success"] = True
3876
+ result["method"] = None
3877
+ result["message"] = "ℹ️ 当前页面未检测到弹窗,无需关闭"
3878
+ result["popup_detected"] = False
3879
+ result["popup_confidence"] = popup_confidence
3880
+ return result
3881
+
3882
+ # ========== 第1步:控件树查找关闭按钮 ==========
3883
+
2900
3884
  # 关闭按钮的常见特征
2901
3885
  close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
2902
3886
  close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
@@ -2975,32 +3959,40 @@ class BasicMobileToolsLite:
2975
3959
  cx, cy = best['center']
2976
3960
  bounds = best['bounds']
2977
3961
 
2978
- # 点击前截图(用于自动学习)
2979
- pre_screenshot = None
2980
- if auto_learn:
2981
- pre_result = self.take_screenshot(description="关闭前", compress=False)
2982
- pre_screenshot = pre_result.get("screenshot_path")
2983
-
2984
- # 点击
2985
- self.click_at_coords(cx, cy)
3962
+ # 点击(click_at_coords 内部已包含应用状态检查和自动返回)
3963
+ click_result = self.click_at_coords(cx, cy)
2986
3964
  time.sleep(0.5)
2987
3965
 
3966
+ # 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
3967
+ app_check = self._check_app_switched()
3968
+ return_result = None
3969
+
3970
+ if app_check['switched']:
3971
+ # 应用已跳转,说明弹窗去除失败,尝试返回目标应用
3972
+ return_result = self._return_to_target_app()
3973
+
2988
3974
  result["success"] = True
2989
3975
  result["method"] = "控件树"
2990
- result["message"] = f"✅ 通过控件树找到关闭按钮并点击\n" \
2991
- f" 位置: ({cx}, {cy})\n" \
2992
- f" 原因: {best['reason']}"
3976
+ msg = f"✅ 通过控件树找到关闭按钮并点击\n" \
3977
+ f" 位置: ({cx}, {cy})\n" \
3978
+ f" 原因: {best['reason']}"
2993
3979
 
2994
- # 自动学习:检查这个 X 是否已在模板库,不在就添加
2995
- if auto_learn and pre_screenshot:
2996
- learn_result = self._auto_learn_template(pre_screenshot, bounds)
2997
- if learn_result:
2998
- result["learned_template"] = learn_result
2999
- result["message"] += f"\n📚 自动学习: {learn_result}"
3980
+ if app_check['switched']:
3981
+ msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
3982
+ if return_result:
3983
+ if return_result['success']:
3984
+ msg += f"\n{return_result['message']}"
3985
+ else:
3986
+ msg += f"\n❌ 自动返回失败: {return_result['message']}"
3987
+
3988
+ result["message"] = msg
3989
+ result["app_check"] = app_check
3990
+ result["return_to_app"] = return_result
3991
+ result["tip"] = "💡 建议调用 mobile_screenshot_with_som 确认弹窗是否已关闭"
3000
3992
 
3001
3993
  return result
3002
3994
 
3003
- # ========== 第2步:模板匹配 ==========
3995
+ # ========== 第2步:模板匹配(自动执行,不需要 AI 介入)==========
3004
3996
  screenshot_path = None
3005
3997
  try:
3006
3998
  from .template_matcher import TemplateMatcher
@@ -3020,15 +4012,30 @@ class BasicMobileToolsLite:
3020
4012
  y_pct = best["percent"]["y"]
3021
4013
 
3022
4014
  # 点击
3023
- self.click_by_percent(x_pct, y_pct)
4015
+ click_result = self.click_by_percent(x_pct, y_pct)
3024
4016
  time.sleep(0.5)
3025
4017
 
4018
+ app_check = self._check_app_switched()
4019
+ return_result = None
4020
+
4021
+ if app_check['switched']:
4022
+ return_result = self._return_to_target_app()
4023
+
3026
4024
  result["success"] = True
3027
4025
  result["method"] = "模板匹配"
3028
- result["message"] = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
3029
- f" 模板: {best.get('template', 'unknown')}\n" \
3030
- f" 置信度: {best.get('confidence', 'N/A')}%\n" \
3031
- f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
4026
+ msg = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
4027
+ f" 模板: {best.get('template', 'unknown')}\n" \
4028
+ f" 置信度: {best.get('confidence', 'N/A')}%\n" \
4029
+ f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
4030
+
4031
+ if app_check['switched']:
4032
+ msg += f"\n⚠️ 应用已跳转"
4033
+ if return_result:
4034
+ msg += f"\n{return_result['message']}"
4035
+
4036
+ result["message"] = msg
4037
+ result["app_check"] = app_check
4038
+ result["return_to_app"] = return_result
3032
4039
  return result
3033
4040
 
3034
4041
  except ImportError:
@@ -3036,17 +4043,12 @@ class BasicMobileToolsLite:
3036
4043
  except Exception:
3037
4044
  pass # 模板匹配失败,继续下一步
3038
4045
 
3039
- # ========== 第3步:返回截图供 AI 分析 ==========
3040
- if not screenshot_path:
3041
- screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
3042
-
4046
+ # ========== 第3步:控件树和模板匹配都失败,提示 AI 使用视觉识别 ==========
3043
4047
  result["success"] = False
4048
+ result["fallback"] = "vision"
3044
4049
  result["method"] = None
3045
- result["message"] = "❌ 控件树和模板匹配都未找到关闭按钮\n" \
3046
- "📸 已截图,请 AI 分析图片中的 X 按钮位置\n" \
3047
- "💡 找到后使用 mobile_click_by_percent(x%, y%) 点击"
3048
- result["screenshot"] = screenshot_result if not screenshot_path else {"screenshot_path": screenshot_path}
3049
- result["need_ai_analysis"] = True
4050
+ result["popup_detected"] = True
4051
+ result["message"] = "⚠️ 控件树和模板匹配都未找到关闭按钮,请调用 mobile_screenshot_with_som 截图后用 click_by_som 点击"
3050
4052
 
3051
4053
  return result
3052
4054