mobile-mcp-ai 2.6.2__py3-none-any.whl → 2.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/config.py +32 -0
- mobile_mcp/core/basic_tools_lite.py +899 -685
- mobile_mcp/mcp_tools/mcp_server.py +259 -250
- {mobile_mcp_ai-2.6.2.dist-info → mobile_mcp_ai-2.6.4.dist-info}/METADATA +17 -30
- {mobile_mcp_ai-2.6.2.dist-info → mobile_mcp_ai-2.6.4.dist-info}/RECORD +9 -9
- {mobile_mcp_ai-2.6.2.dist-info → mobile_mcp_ai-2.6.4.dist-info}/WHEEL +1 -1
- {mobile_mcp_ai-2.6.2.dist-info/licenses → mobile_mcp_ai-2.6.4.dist-info}/LICENSE +0 -0
- {mobile_mcp_ai-2.6.2.dist-info → mobile_mcp_ai-2.6.4.dist-info}/entry_points.txt +0 -0
- {mobile_mcp_ai-2.6.2.dist-info → mobile_mcp_ai-2.6.4.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
- 核心功能精简
|
|
9
9
|
- 保留 pytest 脚本生成
|
|
10
10
|
- 支持操作历史记录
|
|
11
|
+
- Token 优化模式(省钱)
|
|
11
12
|
"""
|
|
12
13
|
|
|
13
14
|
import asyncio
|
|
@@ -17,6 +18,19 @@ from pathlib import Path
|
|
|
17
18
|
from typing import Dict, List, Optional
|
|
18
19
|
from datetime import datetime
|
|
19
20
|
|
|
21
|
+
# Token 优化配置(只精简格式,不限制数量,确保准确度)
|
|
22
|
+
try:
|
|
23
|
+
from mobile_mcp.config import Config
|
|
24
|
+
TOKEN_OPTIMIZATION = Config.TOKEN_OPTIMIZATION_ENABLED
|
|
25
|
+
MAX_ELEMENTS = Config.MAX_ELEMENTS_RETURN
|
|
26
|
+
MAX_SOM_ELEMENTS = Config.MAX_SOM_ELEMENTS_RETURN
|
|
27
|
+
COMPACT_RESPONSE = Config.COMPACT_RESPONSE
|
|
28
|
+
except ImportError:
|
|
29
|
+
TOKEN_OPTIMIZATION = True
|
|
30
|
+
MAX_ELEMENTS = 0 # 0 = 不限制
|
|
31
|
+
MAX_SOM_ELEMENTS = 0 # 0 = 不限制
|
|
32
|
+
COMPACT_RESPONSE = True
|
|
33
|
+
|
|
20
34
|
|
|
21
35
|
class BasicMobileToolsLite:
|
|
22
36
|
"""精简版移动端工具"""
|
|
@@ -48,7 +62,7 @@ class BasicMobileToolsLite:
|
|
|
48
62
|
return None
|
|
49
63
|
|
|
50
64
|
def _record_operation(self, action: str, **kwargs):
|
|
51
|
-
"""
|
|
65
|
+
"""记录操作到历史(旧接口,保持兼容)"""
|
|
52
66
|
record = {
|
|
53
67
|
'action': action,
|
|
54
68
|
'timestamp': datetime.now().isoformat(),
|
|
@@ -56,6 +70,81 @@ class BasicMobileToolsLite:
|
|
|
56
70
|
}
|
|
57
71
|
self.operation_history.append(record)
|
|
58
72
|
|
|
73
|
+
def _record_click(self, locator_type: str, locator_value: str,
|
|
74
|
+
x_percent: float = 0, y_percent: float = 0,
|
|
75
|
+
element_desc: str = '', locator_attr: str = ''):
|
|
76
|
+
"""记录点击操作(标准格式)
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
locator_type: 定位类型 'text' | 'id' | 'percent' | 'coords'
|
|
80
|
+
locator_value: 定位值(文本内容、resource-id、或坐标描述)
|
|
81
|
+
x_percent: 百分比 X 坐标(兜底方案)
|
|
82
|
+
y_percent: 百分比 Y 坐标(兜底方案)
|
|
83
|
+
element_desc: 元素描述(用于脚本注释)
|
|
84
|
+
locator_attr: Android 选择器属性 'text'|'textContains'|'description'|'descriptionContains'
|
|
85
|
+
"""
|
|
86
|
+
record = {
|
|
87
|
+
'action': 'click',
|
|
88
|
+
'timestamp': datetime.now().isoformat(),
|
|
89
|
+
'locator_type': locator_type,
|
|
90
|
+
'locator_value': locator_value,
|
|
91
|
+
'locator_attr': locator_attr or locator_type, # 默认与 type 相同
|
|
92
|
+
'x_percent': x_percent,
|
|
93
|
+
'y_percent': y_percent,
|
|
94
|
+
'element_desc': element_desc or locator_value,
|
|
95
|
+
}
|
|
96
|
+
self.operation_history.append(record)
|
|
97
|
+
|
|
98
|
+
def _record_long_press(self, locator_type: str, locator_value: str,
|
|
99
|
+
duration: float = 1.0,
|
|
100
|
+
x_percent: float = 0, y_percent: float = 0,
|
|
101
|
+
element_desc: str = '', locator_attr: str = ''):
|
|
102
|
+
"""记录长按操作(标准格式)"""
|
|
103
|
+
record = {
|
|
104
|
+
'action': 'long_press',
|
|
105
|
+
'timestamp': datetime.now().isoformat(),
|
|
106
|
+
'locator_type': locator_type,
|
|
107
|
+
'locator_value': locator_value,
|
|
108
|
+
'locator_attr': locator_attr or locator_type,
|
|
109
|
+
'duration': duration,
|
|
110
|
+
'x_percent': x_percent,
|
|
111
|
+
'y_percent': y_percent,
|
|
112
|
+
'element_desc': element_desc or locator_value,
|
|
113
|
+
}
|
|
114
|
+
self.operation_history.append(record)
|
|
115
|
+
|
|
116
|
+
def _record_input(self, text: str, locator_type: str = '', locator_value: str = '',
|
|
117
|
+
x_percent: float = 0, y_percent: float = 0):
|
|
118
|
+
"""记录输入操作(标准格式)"""
|
|
119
|
+
record = {
|
|
120
|
+
'action': 'input',
|
|
121
|
+
'timestamp': datetime.now().isoformat(),
|
|
122
|
+
'text': text,
|
|
123
|
+
'locator_type': locator_type,
|
|
124
|
+
'locator_value': locator_value,
|
|
125
|
+
'x_percent': x_percent,
|
|
126
|
+
'y_percent': y_percent,
|
|
127
|
+
}
|
|
128
|
+
self.operation_history.append(record)
|
|
129
|
+
|
|
130
|
+
def _record_swipe(self, direction: str):
|
|
131
|
+
"""记录滑动操作"""
|
|
132
|
+
record = {
|
|
133
|
+
'action': 'swipe',
|
|
134
|
+
'timestamp': datetime.now().isoformat(),
|
|
135
|
+
'direction': direction,
|
|
136
|
+
}
|
|
137
|
+
self.operation_history.append(record)
|
|
138
|
+
|
|
139
|
+
def _record_key(self, key: str):
|
|
140
|
+
"""记录按键操作"""
|
|
141
|
+
record = {
|
|
142
|
+
'action': 'press_key',
|
|
143
|
+
'timestamp': datetime.now().isoformat(),
|
|
144
|
+
'key': key,
|
|
145
|
+
}
|
|
146
|
+
self.operation_history.append(record)
|
|
147
|
+
|
|
59
148
|
def _get_current_package(self) -> Optional[str]:
|
|
60
149
|
"""获取当前前台应用的包名/Bundle ID"""
|
|
61
150
|
try:
|
|
@@ -260,7 +349,7 @@ class BasicMobileToolsLite:
|
|
|
260
349
|
size = ios_client.wda.window_size()
|
|
261
350
|
screen_width, screen_height = size[0], size[1]
|
|
262
351
|
else:
|
|
263
|
-
return {"success": False, "
|
|
352
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
264
353
|
else:
|
|
265
354
|
self.client.u2.screenshot(str(temp_path))
|
|
266
355
|
info = self.client.u2.info
|
|
@@ -311,22 +400,14 @@ class BasicMobileToolsLite:
|
|
|
311
400
|
|
|
312
401
|
cropped_size = final_path.stat().st_size
|
|
313
402
|
|
|
403
|
+
# 返回结果
|
|
314
404
|
return {
|
|
315
405
|
"success": True,
|
|
316
406
|
"screenshot_path": str(final_path),
|
|
317
|
-
"screen_width": screen_width,
|
|
318
|
-
"screen_height": screen_height,
|
|
319
407
|
"image_width": img.width,
|
|
320
408
|
"image_height": img.height,
|
|
321
409
|
"crop_offset_x": crop_offset_x,
|
|
322
|
-
"crop_offset_y": crop_offset_y
|
|
323
|
-
"file_size": f"{cropped_size/1024:.1f}KB",
|
|
324
|
-
"message": f"🔍 局部截图已保存: {final_path}\n"
|
|
325
|
-
f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
|
|
326
|
-
f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
|
|
327
|
-
f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
|
|
328
|
-
f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
|
|
329
|
-
f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
|
|
410
|
+
"crop_offset_y": crop_offset_y
|
|
330
411
|
}
|
|
331
412
|
|
|
332
413
|
# ========== 情况2:全屏压缩截图 ==========
|
|
@@ -379,24 +460,14 @@ class BasicMobileToolsLite:
|
|
|
379
460
|
compressed_size = final_path.stat().st_size
|
|
380
461
|
saved_percent = (1 - compressed_size / original_size) * 100
|
|
381
462
|
|
|
463
|
+
# 返回结果
|
|
382
464
|
return {
|
|
383
465
|
"success": True,
|
|
384
466
|
"screenshot_path": str(final_path),
|
|
385
|
-
"
|
|
386
|
-
"
|
|
387
|
-
"original_img_width": original_img_width,
|
|
388
|
-
"original_img_height": original_img_height
|
|
389
|
-
"image_width": image_width, # 压缩后宽度(AI 看到的)
|
|
390
|
-
"image_height": image_height, # 压缩后高度(AI 看到的)
|
|
391
|
-
"original_size": f"{original_size/1024:.1f}KB",
|
|
392
|
-
"compressed_size": f"{compressed_size/1024:.1f}KB",
|
|
393
|
-
"saved_percent": f"{saved_percent:.0f}%",
|
|
394
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
395
|
-
f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
|
|
396
|
-
f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
|
|
397
|
-
f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
|
|
398
|
-
f" image_width={image_width}, image_height={image_height},\n"
|
|
399
|
-
f" original_img_width={original_img_width}, original_img_height={original_img_height}"
|
|
467
|
+
"image_width": image_width,
|
|
468
|
+
"image_height": image_height,
|
|
469
|
+
"original_img_width": original_img_width,
|
|
470
|
+
"original_img_height": original_img_height
|
|
400
471
|
}
|
|
401
472
|
|
|
402
473
|
# ========== 情况3:全屏不压缩截图 ==========
|
|
@@ -410,21 +481,12 @@ class BasicMobileToolsLite:
|
|
|
410
481
|
final_path = self.screenshot_dir / filename
|
|
411
482
|
temp_path.rename(final_path)
|
|
412
483
|
|
|
413
|
-
#
|
|
484
|
+
# 返回结果(不压缩时尺寸相同)
|
|
414
485
|
return {
|
|
415
486
|
"success": True,
|
|
416
487
|
"screenshot_path": str(final_path),
|
|
417
|
-
"
|
|
418
|
-
"
|
|
419
|
-
"original_img_width": img.width, # 截图实际尺寸
|
|
420
|
-
"original_img_height": img.height,
|
|
421
|
-
"image_width": img.width, # 未压缩,和原图一样
|
|
422
|
-
"image_height": img.height,
|
|
423
|
-
"file_size": f"{original_size/1024:.1f}KB",
|
|
424
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
425
|
-
f"📐 截图尺寸: {img.width}x{img.height}\n"
|
|
426
|
-
f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
|
|
427
|
-
f"💡 未压缩,坐标可直接使用"
|
|
488
|
+
"image_width": img.width,
|
|
489
|
+
"image_height": img.height
|
|
428
490
|
}
|
|
429
491
|
except ImportError:
|
|
430
492
|
# 如果没有 PIL,回退到原始方式(不压缩)
|
|
@@ -432,7 +494,7 @@ class BasicMobileToolsLite:
|
|
|
432
494
|
except Exception as e:
|
|
433
495
|
return {"success": False, "message": f"❌ 截图失败: {e}"}
|
|
434
496
|
|
|
435
|
-
def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool =
|
|
497
|
+
def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool = False) -> Dict:
|
|
436
498
|
"""截图并添加网格坐标标注(用于精确定位元素)
|
|
437
499
|
|
|
438
500
|
在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
|
|
@@ -464,7 +526,7 @@ class BasicMobileToolsLite:
|
|
|
464
526
|
size = ios_client.wda.window_size()
|
|
465
527
|
screen_width, screen_height = size[0], size[1]
|
|
466
528
|
else:
|
|
467
|
-
return {"success": False, "
|
|
529
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
468
530
|
else:
|
|
469
531
|
self.client.u2.screenshot(str(temp_path))
|
|
470
532
|
info = self.client.u2.info
|
|
@@ -500,7 +562,7 @@ class BasicMobileToolsLite:
|
|
|
500
562
|
# 左侧标注 Y 坐标
|
|
501
563
|
draw.text((2, y + 2), str(y), fill=text_color, font=font_small)
|
|
502
564
|
|
|
503
|
-
# 第3
|
|
565
|
+
# 第3步:检测弹窗并标注(使用严格的置信度检测,避免误识别)
|
|
504
566
|
popup_info = None
|
|
505
567
|
close_positions = []
|
|
506
568
|
|
|
@@ -510,35 +572,12 @@ class BasicMobileToolsLite:
|
|
|
510
572
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
511
573
|
root = ET.fromstring(xml_string)
|
|
512
574
|
|
|
513
|
-
#
|
|
514
|
-
popup_bounds =
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
class_name = elem.attrib.get('class', '')
|
|
518
|
-
|
|
519
|
-
if not bounds_str:
|
|
520
|
-
continue
|
|
521
|
-
|
|
522
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
523
|
-
if not match:
|
|
524
|
-
continue
|
|
525
|
-
|
|
526
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
527
|
-
width = x2 - x1
|
|
528
|
-
height = y2 - y1
|
|
529
|
-
area = width * height
|
|
530
|
-
screen_area = screen_width * screen_height
|
|
531
|
-
|
|
532
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card'])
|
|
533
|
-
area_ratio = area / screen_area if screen_area > 0 else 0
|
|
534
|
-
is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
|
|
535
|
-
is_reasonable_size = 0.08 < area_ratio < 0.85
|
|
536
|
-
|
|
537
|
-
if is_container and is_not_fullscreen and is_reasonable_size and y1 > 50:
|
|
538
|
-
if popup_bounds is None or area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
539
|
-
popup_bounds = (x1, y1, x2, y2)
|
|
575
|
+
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
576
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
577
|
+
root, screen_width, screen_height
|
|
578
|
+
)
|
|
540
579
|
|
|
541
|
-
if popup_bounds:
|
|
580
|
+
if popup_bounds and popup_confidence >= 0.6:
|
|
542
581
|
px1, py1, px2, py2 = popup_bounds
|
|
543
582
|
popup_width = px2 - px1
|
|
544
583
|
popup_height = py2 - py1
|
|
@@ -601,26 +640,16 @@ class BasicMobileToolsLite:
|
|
|
601
640
|
result = {
|
|
602
641
|
"success": True,
|
|
603
642
|
"screenshot_path": str(final_path),
|
|
604
|
-
"screen_width": screen_width,
|
|
605
|
-
"screen_height": screen_height,
|
|
606
643
|
"image_width": img_width,
|
|
607
644
|
"image_height": img_height,
|
|
608
|
-
"grid_size": grid_size
|
|
609
|
-
"message": f"📸 网格截图已保存: {final_path}\n"
|
|
610
|
-
f"📐 尺寸: {img_width}x{img_height}\n"
|
|
611
|
-
f"📏 网格间距: {grid_size}px"
|
|
645
|
+
"grid_size": grid_size
|
|
612
646
|
}
|
|
613
647
|
|
|
614
648
|
if popup_info:
|
|
615
|
-
result["
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
|
|
620
|
-
for pos in close_positions:
|
|
621
|
-
result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
|
|
622
|
-
else:
|
|
623
|
-
result["popup_detected"] = False
|
|
649
|
+
result["popup"] = popup_info["bounds"]
|
|
650
|
+
# 只返回前3个最可能的关闭按钮位置
|
|
651
|
+
if close_positions:
|
|
652
|
+
result["close_hints"] = [(p['x'], p['y']) for p in close_positions[:3]]
|
|
624
653
|
|
|
625
654
|
return result
|
|
626
655
|
|
|
@@ -657,7 +686,7 @@ class BasicMobileToolsLite:
|
|
|
657
686
|
size = ios_client.wda.window_size()
|
|
658
687
|
screen_width, screen_height = size[0], size[1]
|
|
659
688
|
else:
|
|
660
|
-
return {"success": False, "
|
|
689
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
661
690
|
else:
|
|
662
691
|
self.client.u2.screenshot(str(temp_path))
|
|
663
692
|
info = self.client.u2.info
|
|
@@ -766,52 +795,24 @@ class BasicMobileToolsLite:
|
|
|
766
795
|
'index': i + 1,
|
|
767
796
|
'center': (cx, cy),
|
|
768
797
|
'bounds': f"[{x1},{y1}][{x2},{y2}]",
|
|
769
|
-
'desc': elem['desc']
|
|
798
|
+
'desc': elem['desc'],
|
|
799
|
+
'text': elem.get('text', ''),
|
|
800
|
+
'resource_id': elem.get('resource_id', '')
|
|
770
801
|
})
|
|
771
802
|
|
|
772
|
-
# 第3.5
|
|
803
|
+
# 第3.5步:检测弹窗区域(使用严格的置信度检测,避免误识别普通页面)
|
|
773
804
|
popup_bounds = None
|
|
805
|
+
popup_confidence = 0
|
|
774
806
|
|
|
775
807
|
if not self._is_ios():
|
|
776
808
|
try:
|
|
777
|
-
#
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
if not bounds_str:
|
|
783
|
-
continue
|
|
784
|
-
|
|
785
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
786
|
-
if not match:
|
|
787
|
-
continue
|
|
788
|
-
|
|
789
|
-
px1, py1, px2, py2 = map(int, match.groups())
|
|
790
|
-
p_width = px2 - px1
|
|
791
|
-
p_height = py2 - py1
|
|
792
|
-
p_area = p_width * p_height
|
|
793
|
-
screen_area = screen_width * screen_height
|
|
794
|
-
|
|
795
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Frame'])
|
|
796
|
-
area_ratio = p_area / screen_area if screen_area > 0 else 0
|
|
797
|
-
|
|
798
|
-
# 弹窗特征判断(更严格,排除主要内容区域):
|
|
799
|
-
# 1. 不是全屏(宽度和高度都要小于屏幕的95%)
|
|
800
|
-
is_not_fullscreen = (p_width < screen_width * 0.95 and p_height < screen_height * 0.95)
|
|
801
|
-
# 2. 面积范围:10% - 70%(排除主要内容区域,通常占80%+)
|
|
802
|
-
is_reasonable_size = 0.10 < area_ratio < 0.70
|
|
803
|
-
# 3. 不在屏幕左边缘(排除从x=0开始的主要内容容器)
|
|
804
|
-
is_not_at_left_edge = px1 > screen_width * 0.05
|
|
805
|
-
# 4. 高度不能占据屏幕的大部分(排除主要内容区域)
|
|
806
|
-
height_ratio = p_height / screen_height if screen_height > 0 else 0
|
|
807
|
-
is_not_main_content = height_ratio < 0.85
|
|
808
|
-
|
|
809
|
-
if is_container and is_not_fullscreen and is_reasonable_size and is_not_at_left_edge and is_not_main_content and py1 > 30:
|
|
810
|
-
if popup_bounds is None or p_area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
811
|
-
popup_bounds = (px1, py1, px2, py2)
|
|
809
|
+
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
810
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
811
|
+
root, screen_width, screen_height
|
|
812
|
+
)
|
|
812
813
|
|
|
813
814
|
# 如果检测到弹窗,标注弹窗边界(不再猜测X按钮位置)
|
|
814
|
-
if popup_bounds:
|
|
815
|
+
if popup_bounds and popup_confidence >= 0.6:
|
|
815
816
|
px1, py1, px2, py2 = popup_bounds
|
|
816
817
|
|
|
817
818
|
# 只画弹窗边框(蓝色),不再猜测X按钮位置
|
|
@@ -845,38 +846,15 @@ class BasicMobileToolsLite:
|
|
|
845
846
|
img.save(str(final_path), "JPEG", quality=85)
|
|
846
847
|
temp_path.unlink()
|
|
847
848
|
|
|
848
|
-
#
|
|
849
|
-
elements_text = "\n".join([
|
|
850
|
-
f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
|
|
851
|
-
for e in som_elements[:15] # 只显示前15个
|
|
852
|
-
])
|
|
853
|
-
if len(som_elements) > 15:
|
|
854
|
-
elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
|
|
855
|
-
|
|
856
|
-
# 构建弹窗提示文字
|
|
857
|
-
hints_text = ""
|
|
858
|
-
if popup_bounds:
|
|
859
|
-
hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
|
|
860
|
-
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
861
|
-
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
862
|
-
|
|
849
|
+
# 返回结果(Token 优化:不返回 elements 列表,已存储在 self._som_elements)
|
|
863
850
|
return {
|
|
864
851
|
"success": True,
|
|
865
852
|
"screenshot_path": str(final_path),
|
|
866
853
|
"screen_width": screen_width,
|
|
867
854
|
"screen_height": screen_height,
|
|
868
|
-
"image_width": img_width,
|
|
869
|
-
"image_height": img_height,
|
|
870
855
|
"element_count": len(som_elements),
|
|
871
|
-
"elements": som_elements,
|
|
872
856
|
"popup_detected": popup_bounds is not None,
|
|
873
|
-
"
|
|
874
|
-
"message": f"📸 SoM 截图已保存: {final_path}\n"
|
|
875
|
-
f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
876
|
-
f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
877
|
-
f"💡 使用方法:\n"
|
|
878
|
-
f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
879
|
-
f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
857
|
+
"hint": "查看截图上的编号,用 click_by_som(编号) 点击"
|
|
880
858
|
}
|
|
881
859
|
|
|
882
860
|
except ImportError:
|
|
@@ -922,14 +900,41 @@ class BasicMobileToolsLite:
|
|
|
922
900
|
ios_client = self._get_ios_client()
|
|
923
901
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
924
902
|
ios_client.wda.click(cx, cy)
|
|
903
|
+
size = ios_client.wda.window_size()
|
|
904
|
+
screen_width, screen_height = size[0], size[1]
|
|
925
905
|
else:
|
|
926
906
|
self.client.u2.click(cx, cy)
|
|
927
|
-
|
|
907
|
+
info = self.client.u2.info
|
|
908
|
+
screen_width = info.get('displayWidth', 0)
|
|
909
|
+
screen_height = info.get('displayHeight', 0)
|
|
910
|
+
|
|
928
911
|
time.sleep(0.3)
|
|
929
912
|
|
|
913
|
+
# 计算百分比坐标用于跨设备兼容
|
|
914
|
+
x_percent = round(cx / screen_width * 100, 1) if screen_width > 0 else 0
|
|
915
|
+
y_percent = round(cy / screen_height * 100, 1) if screen_height > 0 else 0
|
|
916
|
+
|
|
917
|
+
# 使用标准记录格式
|
|
918
|
+
# 优先使用元素的文本/描述信息,这样生成脚本时可以用文本定位
|
|
919
|
+
elem_text = target.get('text', '')
|
|
920
|
+
elem_id = target.get('resource_id', '')
|
|
921
|
+
elem_desc = target.get('desc', '')
|
|
922
|
+
|
|
923
|
+
if elem_text and not elem_text.startswith('['): # 排除类似 "[可点击]" 的描述
|
|
924
|
+
# 有文本,使用文本定位
|
|
925
|
+
self._record_click('text', elem_text, x_percent, y_percent,
|
|
926
|
+
element_desc=f"[{index}]{elem_desc}", locator_attr='text')
|
|
927
|
+
elif elem_id:
|
|
928
|
+
# 有 resource-id,使用 ID 定位
|
|
929
|
+
self._record_click('id', elem_id, x_percent, y_percent,
|
|
930
|
+
element_desc=f"[{index}]{elem_desc}")
|
|
931
|
+
else:
|
|
932
|
+
# 都没有,使用百分比定位
|
|
933
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
934
|
+
element_desc=f"[{index}]{elem_desc}")
|
|
935
|
+
|
|
930
936
|
return {
|
|
931
937
|
"success": True,
|
|
932
|
-
"message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
|
|
933
938
|
"clicked": {
|
|
934
939
|
"index": index,
|
|
935
940
|
"desc": target['desc'],
|
|
@@ -963,7 +968,7 @@ class BasicMobileToolsLite:
|
|
|
963
968
|
size = ios_client.wda.window_size()
|
|
964
969
|
width, height = size[0], size[1]
|
|
965
970
|
else:
|
|
966
|
-
return {"success": False, "
|
|
971
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
967
972
|
else:
|
|
968
973
|
self.client.u2.screenshot(str(screenshot_path))
|
|
969
974
|
info = self.client.u2.info
|
|
@@ -1041,7 +1046,7 @@ class BasicMobileToolsLite:
|
|
|
1041
1046
|
size = ios_client.wda.window_size()
|
|
1042
1047
|
screen_width, screen_height = size[0], size[1]
|
|
1043
1048
|
else:
|
|
1044
|
-
return {"success": False, "
|
|
1049
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1045
1050
|
else:
|
|
1046
1051
|
info = self.client.u2.info
|
|
1047
1052
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1085,17 +1090,9 @@ class BasicMobileToolsLite:
|
|
|
1085
1090
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1086
1091
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1087
1092
|
|
|
1088
|
-
#
|
|
1089
|
-
self.
|
|
1090
|
-
|
|
1091
|
-
x=x,
|
|
1092
|
-
y=y,
|
|
1093
|
-
x_percent=x_percent,
|
|
1094
|
-
y_percent=y_percent,
|
|
1095
|
-
screen_width=screen_width,
|
|
1096
|
-
screen_height=screen_height,
|
|
1097
|
-
ref=f"coords_{x}_{y}"
|
|
1098
|
-
)
|
|
1093
|
+
# 使用标准记录格式:坐标点击用百分比作为定位方式(跨分辨率兼容)
|
|
1094
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
1095
|
+
element_desc=f"坐标({x},{y})")
|
|
1099
1096
|
|
|
1100
1097
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1101
1098
|
app_check = self._check_app_switched()
|
|
@@ -1164,14 +1161,14 @@ class BasicMobileToolsLite:
|
|
|
1164
1161
|
size = ios_client.wda.window_size()
|
|
1165
1162
|
width, height = size[0], size[1]
|
|
1166
1163
|
else:
|
|
1167
|
-
return {"success": False, "
|
|
1164
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1168
1165
|
else:
|
|
1169
1166
|
info = self.client.u2.info
|
|
1170
1167
|
width = info.get('displayWidth', 0)
|
|
1171
1168
|
height = info.get('displayHeight', 0)
|
|
1172
1169
|
|
|
1173
1170
|
if width == 0 or height == 0:
|
|
1174
|
-
return {"success": False, "
|
|
1171
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1175
1172
|
|
|
1176
1173
|
# 第2步:百分比转像素坐标
|
|
1177
1174
|
# 公式:像素 = 屏幕尺寸 × (百分比 / 100)
|
|
@@ -1186,23 +1183,12 @@ class BasicMobileToolsLite:
|
|
|
1186
1183
|
|
|
1187
1184
|
time.sleep(0.3)
|
|
1188
1185
|
|
|
1189
|
-
# 第4
|
|
1190
|
-
self.
|
|
1191
|
-
|
|
1192
|
-
x=x,
|
|
1193
|
-
y=y,
|
|
1194
|
-
x_percent=x_percent,
|
|
1195
|
-
y_percent=y_percent,
|
|
1196
|
-
screen_width=width,
|
|
1197
|
-
screen_height=height,
|
|
1198
|
-
ref=f"percent_{x_percent}_{y_percent}"
|
|
1199
|
-
)
|
|
1186
|
+
# 第4步:使用标准记录格式
|
|
1187
|
+
self._record_click('percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent,
|
|
1188
|
+
element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1200
1189
|
|
|
1201
1190
|
return {
|
|
1202
1191
|
"success": True,
|
|
1203
|
-
"message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
|
|
1204
|
-
"screen_size": {"width": width, "height": height},
|
|
1205
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1206
1192
|
"pixel": {"x": x, "y": y}
|
|
1207
1193
|
}
|
|
1208
1194
|
except Exception as e:
|
|
@@ -1228,10 +1214,16 @@ class BasicMobileToolsLite:
|
|
|
1228
1214
|
if elem.exists:
|
|
1229
1215
|
elem.click()
|
|
1230
1216
|
time.sleep(0.3)
|
|
1231
|
-
self.
|
|
1232
|
-
return {"success": True
|
|
1233
|
-
|
|
1217
|
+
self._record_click('text', text, element_desc=text, locator_attr='text')
|
|
1218
|
+
return {"success": True}
|
|
1219
|
+
# 控件树找不到,提示用视觉识别
|
|
1220
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1221
|
+
else:
|
|
1222
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1234
1223
|
else:
|
|
1224
|
+
# 获取屏幕尺寸用于计算百分比
|
|
1225
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
1226
|
+
|
|
1235
1227
|
# 🔍 先查 XML 树,找到元素及其属性
|
|
1236
1228
|
found_elem = self._find_element_in_tree(text, position=position)
|
|
1237
1229
|
|
|
@@ -1240,15 +1232,23 @@ class BasicMobileToolsLite:
|
|
|
1240
1232
|
attr_value = found_elem['attr_value']
|
|
1241
1233
|
bounds = found_elem.get('bounds')
|
|
1242
1234
|
|
|
1243
|
-
#
|
|
1235
|
+
# 计算百分比坐标作为兜底
|
|
1236
|
+
x_pct, y_pct = 0, 0
|
|
1237
|
+
if bounds:
|
|
1238
|
+
cx = (bounds[0] + bounds[2]) // 2
|
|
1239
|
+
cy = (bounds[1] + bounds[3]) // 2
|
|
1240
|
+
x_pct = round(cx / screen_width * 100, 1)
|
|
1241
|
+
y_pct = round(cy / screen_height * 100, 1)
|
|
1242
|
+
|
|
1243
|
+
# 如果有位置参数,直接使用坐标点击
|
|
1244
1244
|
if position and bounds:
|
|
1245
1245
|
x = (bounds[0] + bounds[2]) // 2
|
|
1246
1246
|
y = (bounds[1] + bounds[3]) // 2
|
|
1247
1247
|
self.client.u2.click(x, y)
|
|
1248
1248
|
time.sleep(0.3)
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
return {"success": True
|
|
1249
|
+
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1250
|
+
element_desc=f"{text}({position})", locator_attr=attr_type)
|
|
1251
|
+
return {"success": True}
|
|
1252
1252
|
|
|
1253
1253
|
# 没有位置参数时,使用选择器定位
|
|
1254
1254
|
if attr_type == 'text':
|
|
@@ -1265,23 +1265,24 @@ class BasicMobileToolsLite:
|
|
|
1265
1265
|
if elem and elem.exists(timeout=1):
|
|
1266
1266
|
elem.click()
|
|
1267
1267
|
time.sleep(0.3)
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
return {"success": True
|
|
1268
|
+
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1269
|
+
element_desc=text, locator_attr=attr_type)
|
|
1270
|
+
return {"success": True}
|
|
1271
1271
|
|
|
1272
|
-
#
|
|
1272
|
+
# 选择器失败,用坐标兜底
|
|
1273
1273
|
if bounds:
|
|
1274
1274
|
x = (bounds[0] + bounds[2]) // 2
|
|
1275
1275
|
y = (bounds[1] + bounds[3]) // 2
|
|
1276
1276
|
self.client.u2.click(x, y)
|
|
1277
1277
|
time.sleep(0.3)
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
return {"success": True
|
|
1278
|
+
self._record_click('percent', f"{x_pct}%,{y_pct}%", x_pct, y_pct,
|
|
1279
|
+
element_desc=text)
|
|
1280
|
+
return {"success": True}
|
|
1281
1281
|
|
|
1282
|
-
|
|
1282
|
+
# 控件树找不到,提示用视觉识别
|
|
1283
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1283
1284
|
except Exception as e:
|
|
1284
|
-
return {"success": False, "
|
|
1285
|
+
return {"success": False, "msg": str(e)}
|
|
1285
1286
|
|
|
1286
1287
|
def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
|
|
1287
1288
|
"""在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
|
|
@@ -1420,15 +1421,8 @@ class BasicMobileToolsLite:
|
|
|
1420
1421
|
return None
|
|
1421
1422
|
|
|
1422
1423
|
def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
|
|
1423
|
-
"""通过 resource-id
|
|
1424
|
-
|
|
1425
|
-
Args:
|
|
1426
|
-
resource_id: 元素的 resource-id
|
|
1427
|
-
index: 第几个元素(从 0 开始),默认 0 表示第一个
|
|
1428
|
-
"""
|
|
1424
|
+
"""通过 resource-id 点击"""
|
|
1429
1425
|
try:
|
|
1430
|
-
index_desc = f"[{index}]" if index > 0 else ""
|
|
1431
|
-
|
|
1432
1426
|
if self._is_ios():
|
|
1433
1427
|
ios_client = self._get_ios_client()
|
|
1434
1428
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
@@ -1436,31 +1430,31 @@ class BasicMobileToolsLite:
|
|
|
1436
1430
|
if not elem.exists:
|
|
1437
1431
|
elem = ios_client.wda(name=resource_id)
|
|
1438
1432
|
if elem.exists:
|
|
1439
|
-
# 获取所有匹配的元素
|
|
1440
1433
|
elements = elem.find_elements()
|
|
1441
1434
|
if index < len(elements):
|
|
1442
1435
|
elements[index].click()
|
|
1443
1436
|
time.sleep(0.3)
|
|
1444
|
-
self.
|
|
1445
|
-
return {"success": True
|
|
1437
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1438
|
+
return {"success": True}
|
|
1446
1439
|
else:
|
|
1447
|
-
return {"success": False, "
|
|
1448
|
-
return {"success": False, "
|
|
1440
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{len(elements)}个)"}
|
|
1441
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1442
|
+
else:
|
|
1443
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1449
1444
|
else:
|
|
1450
1445
|
elem = self.client.u2(resourceId=resource_id)
|
|
1451
1446
|
if elem.exists(timeout=0.5):
|
|
1452
|
-
# 获取匹配元素数量
|
|
1453
1447
|
count = elem.count
|
|
1454
1448
|
if index < count:
|
|
1455
1449
|
elem[index].click()
|
|
1456
1450
|
time.sleep(0.3)
|
|
1457
|
-
self.
|
|
1458
|
-
return {"success": True
|
|
1451
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1452
|
+
return {"success": True}
|
|
1459
1453
|
else:
|
|
1460
|
-
return {"success": False, "
|
|
1461
|
-
return {"success": False, "
|
|
1454
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{count}个)"}
|
|
1455
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1462
1456
|
except Exception as e:
|
|
1463
|
-
return {"success": False, "
|
|
1457
|
+
return {"success": False, "msg": str(e)}
|
|
1464
1458
|
|
|
1465
1459
|
# ==================== 长按操作 ====================
|
|
1466
1460
|
|
|
@@ -1494,7 +1488,7 @@ class BasicMobileToolsLite:
|
|
|
1494
1488
|
size = ios_client.wda.window_size()
|
|
1495
1489
|
screen_width, screen_height = size[0], size[1]
|
|
1496
1490
|
else:
|
|
1497
|
-
return {"success": False, "
|
|
1491
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1498
1492
|
else:
|
|
1499
1493
|
info = self.client.u2.info
|
|
1500
1494
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1541,38 +1535,17 @@ class BasicMobileToolsLite:
|
|
|
1541
1535
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1542
1536
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1543
1537
|
|
|
1544
|
-
#
|
|
1545
|
-
self.
|
|
1546
|
-
|
|
1547
|
-
x=x,
|
|
1548
|
-
y=y,
|
|
1549
|
-
x_percent=x_percent,
|
|
1550
|
-
y_percent=y_percent,
|
|
1551
|
-
duration=duration,
|
|
1552
|
-
screen_width=screen_width,
|
|
1553
|
-
screen_height=screen_height,
|
|
1554
|
-
ref=f"coords_{x}_{y}"
|
|
1555
|
-
)
|
|
1538
|
+
# 使用标准记录格式
|
|
1539
|
+
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1540
|
+
x_percent, y_percent, element_desc=f"坐标({x},{y})")
|
|
1556
1541
|
|
|
1557
1542
|
if converted:
|
|
1558
1543
|
if conversion_type == "crop_offset":
|
|
1559
|
-
return {
|
|
1560
|
-
"success": True,
|
|
1561
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1562
|
-
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
1563
|
-
}
|
|
1544
|
+
return {"success": True}
|
|
1564
1545
|
else:
|
|
1565
|
-
return {
|
|
1566
|
-
"success": True,
|
|
1567
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1568
|
-
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
1569
|
-
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
1570
|
-
}
|
|
1546
|
+
return {"success": True}
|
|
1571
1547
|
else:
|
|
1572
|
-
return {
|
|
1573
|
-
"success": True,
|
|
1574
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
|
|
1575
|
-
}
|
|
1548
|
+
return {"success": True}
|
|
1576
1549
|
except Exception as e:
|
|
1577
1550
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1578
1551
|
|
|
@@ -1601,14 +1574,14 @@ class BasicMobileToolsLite:
|
|
|
1601
1574
|
size = ios_client.wda.window_size()
|
|
1602
1575
|
width, height = size[0], size[1]
|
|
1603
1576
|
else:
|
|
1604
|
-
return {"success": False, "
|
|
1577
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1605
1578
|
else:
|
|
1606
1579
|
info = self.client.u2.info
|
|
1607
1580
|
width = info.get('displayWidth', 0)
|
|
1608
1581
|
height = info.get('displayHeight', 0)
|
|
1609
1582
|
|
|
1610
1583
|
if width == 0 or height == 0:
|
|
1611
|
-
return {"success": False, "
|
|
1584
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1612
1585
|
|
|
1613
1586
|
# 第2步:百分比转像素坐标
|
|
1614
1587
|
x = int(width * x_percent / 100)
|
|
@@ -1626,26 +1599,11 @@ class BasicMobileToolsLite:
|
|
|
1626
1599
|
|
|
1627
1600
|
time.sleep(0.3)
|
|
1628
1601
|
|
|
1629
|
-
# 第4
|
|
1630
|
-
self.
|
|
1631
|
-
|
|
1632
|
-
x=x,
|
|
1633
|
-
y=y,
|
|
1634
|
-
x_percent=x_percent,
|
|
1635
|
-
y_percent=y_percent,
|
|
1636
|
-
duration=duration,
|
|
1637
|
-
screen_width=width,
|
|
1638
|
-
screen_height=height,
|
|
1639
|
-
ref=f"percent_{x_percent}_{y_percent}"
|
|
1640
|
-
)
|
|
1602
|
+
# 第4步:使用标准记录格式
|
|
1603
|
+
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1604
|
+
x_percent, y_percent, element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1641
1605
|
|
|
1642
|
-
return {
|
|
1643
|
-
"success": True,
|
|
1644
|
-
"message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
|
|
1645
|
-
"screen_size": {"width": width, "height": height},
|
|
1646
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1647
|
-
"pixel": {"x": x, "y": y},
|
|
1648
|
-
"duration": duration
|
|
1606
|
+
return {"success": True
|
|
1649
1607
|
}
|
|
1650
1608
|
except Exception as e:
|
|
1651
1609
|
return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
|
|
@@ -1674,10 +1632,13 @@ class BasicMobileToolsLite:
|
|
|
1674
1632
|
else:
|
|
1675
1633
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1676
1634
|
time.sleep(0.3)
|
|
1677
|
-
self.
|
|
1678
|
-
return {"success": True
|
|
1679
|
-
return {"success": False, "
|
|
1635
|
+
self._record_long_press('text', text, duration, element_desc=text, locator_attr='text')
|
|
1636
|
+
return {"success": True}
|
|
1637
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1680
1638
|
else:
|
|
1639
|
+
# 获取屏幕尺寸用于计算百分比
|
|
1640
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
1641
|
+
|
|
1681
1642
|
# 先查 XML 树,找到元素
|
|
1682
1643
|
found_elem = self._find_element_in_tree(text)
|
|
1683
1644
|
|
|
@@ -1686,6 +1647,14 @@ class BasicMobileToolsLite:
|
|
|
1686
1647
|
attr_value = found_elem['attr_value']
|
|
1687
1648
|
bounds = found_elem.get('bounds')
|
|
1688
1649
|
|
|
1650
|
+
# 计算百分比坐标作为兜底
|
|
1651
|
+
x_pct, y_pct = 0, 0
|
|
1652
|
+
if bounds:
|
|
1653
|
+
cx = (bounds[0] + bounds[2]) // 2
|
|
1654
|
+
cy = (bounds[1] + bounds[3]) // 2
|
|
1655
|
+
x_pct = round(cx / screen_width * 100, 1)
|
|
1656
|
+
y_pct = round(cy / screen_height * 100, 1)
|
|
1657
|
+
|
|
1689
1658
|
# 根据找到的属性类型,使用对应的选择器
|
|
1690
1659
|
if attr_type == 'text':
|
|
1691
1660
|
elem = self.client.u2(text=attr_value)
|
|
@@ -1701,8 +1670,9 @@ class BasicMobileToolsLite:
|
|
|
1701
1670
|
if elem and elem.exists(timeout=1):
|
|
1702
1671
|
elem.long_click(duration=duration)
|
|
1703
1672
|
time.sleep(0.3)
|
|
1704
|
-
self.
|
|
1705
|
-
|
|
1673
|
+
self._record_long_press('text', attr_value, duration, x_pct, y_pct,
|
|
1674
|
+
element_desc=text, locator_attr=attr_type)
|
|
1675
|
+
return {"success": True}
|
|
1706
1676
|
|
|
1707
1677
|
# 如果选择器失败,用坐标兜底
|
|
1708
1678
|
if bounds:
|
|
@@ -1710,10 +1680,11 @@ class BasicMobileToolsLite:
|
|
|
1710
1680
|
y = (bounds[1] + bounds[3]) // 2
|
|
1711
1681
|
self.client.u2.long_click(x, y, duration=duration)
|
|
1712
1682
|
time.sleep(0.3)
|
|
1713
|
-
self.
|
|
1714
|
-
|
|
1683
|
+
self._record_long_press('percent', f"{x_pct}%,{y_pct}%", duration, x_pct, y_pct,
|
|
1684
|
+
element_desc=text)
|
|
1685
|
+
return {"success": True}
|
|
1715
1686
|
|
|
1716
|
-
return {"success": False, "
|
|
1687
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1717
1688
|
except Exception as e:
|
|
1718
1689
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1719
1690
|
|
|
@@ -1740,17 +1711,17 @@ class BasicMobileToolsLite:
|
|
|
1740
1711
|
else:
|
|
1741
1712
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1742
1713
|
time.sleep(0.3)
|
|
1743
|
-
self.
|
|
1744
|
-
return {"success": True
|
|
1745
|
-
return {"success": False, "
|
|
1714
|
+
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1715
|
+
return {"success": True}
|
|
1716
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1746
1717
|
else:
|
|
1747
1718
|
elem = self.client.u2(resourceId=resource_id)
|
|
1748
1719
|
if elem.exists(timeout=0.5):
|
|
1749
1720
|
elem.long_click(duration=duration)
|
|
1750
1721
|
time.sleep(0.3)
|
|
1751
|
-
self.
|
|
1722
|
+
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1752
1723
|
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
1753
|
-
return {"success": False, "
|
|
1724
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1754
1725
|
except Exception as e:
|
|
1755
1726
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1756
1727
|
|
|
@@ -1775,7 +1746,7 @@ class BasicMobileToolsLite:
|
|
|
1775
1746
|
if elem.exists:
|
|
1776
1747
|
elem.set_text(text)
|
|
1777
1748
|
time.sleep(0.3)
|
|
1778
|
-
self.
|
|
1749
|
+
self._record_input(text, 'id', resource_id)
|
|
1779
1750
|
|
|
1780
1751
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1781
1752
|
app_check = self._check_app_switched()
|
|
@@ -1810,7 +1781,7 @@ class BasicMobileToolsLite:
|
|
|
1810
1781
|
if count == 1:
|
|
1811
1782
|
elements.set_text(text)
|
|
1812
1783
|
time.sleep(0.3)
|
|
1813
|
-
self.
|
|
1784
|
+
self._record_input(text, 'id', resource_id)
|
|
1814
1785
|
|
|
1815
1786
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1816
1787
|
app_check = self._check_app_switched()
|
|
@@ -1844,7 +1815,7 @@ class BasicMobileToolsLite:
|
|
|
1844
1815
|
if info.get('editable') or info.get('focusable'):
|
|
1845
1816
|
elem.set_text(text)
|
|
1846
1817
|
time.sleep(0.3)
|
|
1847
|
-
self.
|
|
1818
|
+
self._record_input(text, 'id', resource_id)
|
|
1848
1819
|
|
|
1849
1820
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1850
1821
|
app_check = self._check_app_switched()
|
|
@@ -1872,7 +1843,7 @@ class BasicMobileToolsLite:
|
|
|
1872
1843
|
# 没找到可编辑的,用第一个
|
|
1873
1844
|
elements[0].set_text(text)
|
|
1874
1845
|
time.sleep(0.3)
|
|
1875
|
-
self.
|
|
1846
|
+
self._record_input(text, 'id', resource_id)
|
|
1876
1847
|
|
|
1877
1848
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1878
1849
|
app_check = self._check_app_switched()
|
|
@@ -1903,7 +1874,7 @@ class BasicMobileToolsLite:
|
|
|
1903
1874
|
if et_count == 1:
|
|
1904
1875
|
edit_texts.set_text(text)
|
|
1905
1876
|
time.sleep(0.3)
|
|
1906
|
-
self.
|
|
1877
|
+
self._record_input(text, 'class', 'EditText')
|
|
1907
1878
|
|
|
1908
1879
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1909
1880
|
app_check = self._check_app_switched()
|
|
@@ -1943,7 +1914,7 @@ class BasicMobileToolsLite:
|
|
|
1943
1914
|
if best_elem:
|
|
1944
1915
|
best_elem.set_text(text)
|
|
1945
1916
|
time.sleep(0.3)
|
|
1946
|
-
self.
|
|
1917
|
+
self._record_input(text, 'class', 'EditText')
|
|
1947
1918
|
|
|
1948
1919
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
1949
1920
|
app_check = self._check_app_switched()
|
|
@@ -2007,15 +1978,8 @@ class BasicMobileToolsLite:
|
|
|
2007
1978
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
2008
1979
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
2009
1980
|
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
x=x,
|
|
2013
|
-
y=y,
|
|
2014
|
-
x_percent=x_percent,
|
|
2015
|
-
y_percent=y_percent,
|
|
2016
|
-
ref=f"coords_{x}_{y}",
|
|
2017
|
-
text=text
|
|
2018
|
-
)
|
|
1981
|
+
# 使用标准记录格式
|
|
1982
|
+
self._record_input(text, 'percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent)
|
|
2019
1983
|
|
|
2020
1984
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2021
1985
|
app_check = self._check_app_switched()
|
|
@@ -2060,7 +2024,7 @@ class BasicMobileToolsLite:
|
|
|
2060
2024
|
size = ios_client.wda.window_size()
|
|
2061
2025
|
width, height = size[0], size[1]
|
|
2062
2026
|
else:
|
|
2063
|
-
return {"success": False, "
|
|
2027
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
2064
2028
|
else:
|
|
2065
2029
|
width, height = self.client.u2.window_size()
|
|
2066
2030
|
|
|
@@ -2098,13 +2062,8 @@ class BasicMobileToolsLite:
|
|
|
2098
2062
|
else:
|
|
2099
2063
|
self.client.u2.swipe(x1, y1, x2, y2, duration=0.5)
|
|
2100
2064
|
|
|
2101
|
-
#
|
|
2102
|
-
|
|
2103
|
-
if y is not None:
|
|
2104
|
-
record_info['y'] = y
|
|
2105
|
-
if y_percent is not None:
|
|
2106
|
-
record_info['y_percent'] = y_percent
|
|
2107
|
-
self._record_operation('swipe', **record_info)
|
|
2065
|
+
# 使用标准记录格式
|
|
2066
|
+
self._record_swipe(direction)
|
|
2108
2067
|
|
|
2109
2068
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2110
2069
|
app_check = self._check_app_switched()
|
|
@@ -2161,22 +2120,22 @@ class BasicMobileToolsLite:
|
|
|
2161
2120
|
ios_client.wda.send_keys('\n')
|
|
2162
2121
|
elif ios_key == 'home':
|
|
2163
2122
|
ios_client.wda.home()
|
|
2164
|
-
return {"success": True
|
|
2165
|
-
return {"success": False, "
|
|
2123
|
+
return {"success": True}
|
|
2124
|
+
return {"success": False, "msg": f"iOS不支持{key}"}
|
|
2166
2125
|
else:
|
|
2167
2126
|
keycode = key_map.get(key.lower())
|
|
2168
2127
|
if keycode:
|
|
2169
2128
|
self.client.u2.shell(f'input keyevent {keycode}')
|
|
2170
|
-
self.
|
|
2171
|
-
return {"success": True
|
|
2172
|
-
return {"success": False, "
|
|
2129
|
+
self._record_key(key)
|
|
2130
|
+
return {"success": True}
|
|
2131
|
+
return {"success": False, "msg": f"不支持按键{key}"}
|
|
2173
2132
|
except Exception as e:
|
|
2174
2133
|
return {"success": False, "message": f"❌ 按键失败: {e}"}
|
|
2175
2134
|
|
|
2176
2135
|
def wait(self, seconds: float) -> Dict:
|
|
2177
2136
|
"""等待指定时间"""
|
|
2178
2137
|
time.sleep(seconds)
|
|
2179
|
-
return {"success": True
|
|
2138
|
+
return {"success": True}
|
|
2180
2139
|
|
|
2181
2140
|
# ==================== 应用管理 ====================
|
|
2182
2141
|
|
|
@@ -2205,10 +2164,7 @@ class BasicMobileToolsLite:
|
|
|
2205
2164
|
|
|
2206
2165
|
self._record_operation('launch_app', package_name=package_name)
|
|
2207
2166
|
|
|
2208
|
-
return {
|
|
2209
|
-
"success": True,
|
|
2210
|
-
"message": f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载\n📱 已设置应用状态监测"
|
|
2211
|
-
}
|
|
2167
|
+
return {"success": True}
|
|
2212
2168
|
except Exception as e:
|
|
2213
2169
|
return {"success": False, "message": f"❌ 启动失败: {e}"}
|
|
2214
2170
|
|
|
@@ -2221,9 +2177,9 @@ class BasicMobileToolsLite:
|
|
|
2221
2177
|
ios_client.wda.app_terminate(package_name)
|
|
2222
2178
|
else:
|
|
2223
2179
|
self.client.u2.app_stop(package_name)
|
|
2224
|
-
return {"success": True
|
|
2180
|
+
return {"success": True}
|
|
2225
2181
|
except Exception as e:
|
|
2226
|
-
return {"success": False, "
|
|
2182
|
+
return {"success": False, "msg": str(e)}
|
|
2227
2183
|
|
|
2228
2184
|
def list_apps(self, filter_keyword: str = "") -> Dict:
|
|
2229
2185
|
"""列出已安装应用"""
|
|
@@ -2335,6 +2291,26 @@ class BasicMobileToolsLite:
|
|
|
2335
2291
|
'_shadow', 'shadow_', '_divider', 'divider_', '_line', 'line_'
|
|
2336
2292
|
}
|
|
2337
2293
|
|
|
2294
|
+
# Token 优化:构建精简元素(只返回非空字段)
|
|
2295
|
+
def build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name):
|
|
2296
|
+
"""只返回有值的字段,节省 token"""
|
|
2297
|
+
item = {}
|
|
2298
|
+
if resource_id:
|
|
2299
|
+
# 精简 resource_id,只保留最后一段
|
|
2300
|
+
item['id'] = resource_id.split('/')[-1] if '/' in resource_id else resource_id
|
|
2301
|
+
if text:
|
|
2302
|
+
item['text'] = text
|
|
2303
|
+
if content_desc:
|
|
2304
|
+
item['desc'] = content_desc
|
|
2305
|
+
if bounds:
|
|
2306
|
+
item['bounds'] = bounds
|
|
2307
|
+
if likely_click:
|
|
2308
|
+
item['click'] = True # 启发式判断可点击
|
|
2309
|
+
# class 精简:只保留关键类型
|
|
2310
|
+
if class_name in ('EditText', 'TextInput', 'Button', 'ImageButton', 'CheckBox', 'Switch'):
|
|
2311
|
+
item['type'] = class_name
|
|
2312
|
+
return item
|
|
2313
|
+
|
|
2338
2314
|
result = []
|
|
2339
2315
|
for elem in elements:
|
|
2340
2316
|
# 获取元素属性
|
|
@@ -2354,14 +2330,11 @@ class BasicMobileToolsLite:
|
|
|
2354
2330
|
|
|
2355
2331
|
# 2. 检查是否是功能控件(直接保留)
|
|
2356
2332
|
if class_name in FUNCTIONAL_WIDGETS:
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
'clickable': clickable,
|
|
2363
|
-
'class': class_name
|
|
2364
|
-
})
|
|
2333
|
+
# 使用启发式判断可点击性(替代不准确的 clickable 属性)
|
|
2334
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2335
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2336
|
+
if item:
|
|
2337
|
+
result.append(item)
|
|
2365
2338
|
continue
|
|
2366
2339
|
|
|
2367
2340
|
# 3. 检查是否是容器控件
|
|
@@ -2374,14 +2347,10 @@ class BasicMobileToolsLite:
|
|
|
2374
2347
|
# 所有属性都是默认值,过滤掉
|
|
2375
2348
|
continue
|
|
2376
2349
|
# 有业务ID或其他有意义属性,保留
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
'bounds': bounds,
|
|
2382
|
-
'clickable': clickable,
|
|
2383
|
-
'class': class_name
|
|
2384
|
-
})
|
|
2350
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2351
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2352
|
+
if item:
|
|
2353
|
+
result.append(item)
|
|
2385
2354
|
continue
|
|
2386
2355
|
|
|
2387
2356
|
# 4. 检查是否是装饰类控件
|
|
@@ -2398,14 +2367,21 @@ class BasicMobileToolsLite:
|
|
|
2398
2367
|
continue
|
|
2399
2368
|
|
|
2400
2369
|
# 6. 其他情况:有意义的元素保留
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2370
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2371
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2372
|
+
if item:
|
|
2373
|
+
result.append(item)
|
|
2374
|
+
|
|
2375
|
+
# Token 优化:可选限制返回元素数量(默认不限制,确保准确度)
|
|
2376
|
+
if TOKEN_OPTIMIZATION and MAX_ELEMENTS > 0 and len(result) > MAX_ELEMENTS:
|
|
2377
|
+
# 仅在用户明确设置 MAX_ELEMENTS_RETURN 时才截断
|
|
2378
|
+
truncated = result[:MAX_ELEMENTS]
|
|
2379
|
+
truncated.append({
|
|
2380
|
+
'_truncated': True,
|
|
2381
|
+
'_total': len(result),
|
|
2382
|
+
'_shown': MAX_ELEMENTS
|
|
2408
2383
|
})
|
|
2384
|
+
return truncated
|
|
2409
2385
|
|
|
2410
2386
|
return result
|
|
2411
2387
|
except Exception as e:
|
|
@@ -2442,6 +2418,68 @@ class BasicMobileToolsLite:
|
|
|
2442
2418
|
|
|
2443
2419
|
return True
|
|
2444
2420
|
|
|
2421
|
+
def _is_likely_clickable(self, class_name: str, resource_id: str, text: str,
|
|
2422
|
+
content_desc: str, clickable: bool, bounds: str) -> bool:
|
|
2423
|
+
"""
|
|
2424
|
+
启发式判断元素是否可能可点击
|
|
2425
|
+
|
|
2426
|
+
Android 的 clickable 属性经常不准确,因为:
|
|
2427
|
+
1. 点击事件可能设置在父容器上
|
|
2428
|
+
2. 使用 onTouchListener 而不是 onClick
|
|
2429
|
+
3. RecyclerView item 通过 ItemClickListener 处理
|
|
2430
|
+
|
|
2431
|
+
此方法通过多种规则推断元素的真实可点击性
|
|
2432
|
+
"""
|
|
2433
|
+
# 规则1:clickable=true 肯定可点击
|
|
2434
|
+
if clickable:
|
|
2435
|
+
return True
|
|
2436
|
+
|
|
2437
|
+
# 规则2:特定类型的控件通常可点击
|
|
2438
|
+
TYPICALLY_CLICKABLE = {
|
|
2439
|
+
'Button', 'ImageButton', 'CheckBox', 'RadioButton', 'Switch',
|
|
2440
|
+
'ToggleButton', 'FloatingActionButton', 'Chip', 'TabView',
|
|
2441
|
+
'EditText', 'TextInput', # 输入框可点击获取焦点
|
|
2442
|
+
}
|
|
2443
|
+
if class_name in TYPICALLY_CLICKABLE:
|
|
2444
|
+
return True
|
|
2445
|
+
|
|
2446
|
+
# 规则3:resource_id 包含可点击关键词
|
|
2447
|
+
if resource_id:
|
|
2448
|
+
id_lower = resource_id.lower()
|
|
2449
|
+
CLICK_KEYWORDS = [
|
|
2450
|
+
'btn', 'button', 'click', 'tap', 'submit', 'confirm',
|
|
2451
|
+
'cancel', 'close', 'back', 'next', 'prev', 'more',
|
|
2452
|
+
'action', 'link', 'menu', 'tab', 'item', 'cell',
|
|
2453
|
+
'card', 'avatar', 'icon', 'entry', 'option', 'arrow'
|
|
2454
|
+
]
|
|
2455
|
+
for kw in CLICK_KEYWORDS:
|
|
2456
|
+
if kw in id_lower:
|
|
2457
|
+
return True
|
|
2458
|
+
|
|
2459
|
+
# 规则4:content_desc 包含可点击暗示
|
|
2460
|
+
if content_desc:
|
|
2461
|
+
desc_lower = content_desc.lower()
|
|
2462
|
+
CLICK_HINTS = ['点击', '按钮', '关闭', '返回', '更多', 'click', 'tap', 'button', 'close']
|
|
2463
|
+
for hint in CLICK_HINTS:
|
|
2464
|
+
if hint in desc_lower:
|
|
2465
|
+
return True
|
|
2466
|
+
|
|
2467
|
+
# 规则5:有 resource_id 或 content_desc 的小图标可能可点击
|
|
2468
|
+
# (纯 ImageView 不加判断,误判率太高)
|
|
2469
|
+
if class_name in ('ImageView', 'Image') and (resource_id or content_desc) and bounds:
|
|
2470
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
2471
|
+
if match:
|
|
2472
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2473
|
+
w, h = x2 - x1, y2 - y1
|
|
2474
|
+
# 小图标(20-100px)更可能是按钮
|
|
2475
|
+
if 20 <= w <= 100 and 20 <= h <= 100:
|
|
2476
|
+
return True
|
|
2477
|
+
|
|
2478
|
+
# 规则6:移除(TextView 误判率太高,只依赖上面的规则)
|
|
2479
|
+
# 如果有 clickable=true 或 ID/desc 中有关键词,前面的规则已经覆盖
|
|
2480
|
+
|
|
2481
|
+
return False
|
|
2482
|
+
|
|
2445
2483
|
def find_close_button(self) -> Dict:
|
|
2446
2484
|
"""智能查找关闭按钮(不点击,只返回位置)
|
|
2447
2485
|
|
|
@@ -2455,7 +2493,7 @@ class BasicMobileToolsLite:
|
|
|
2455
2493
|
import re
|
|
2456
2494
|
|
|
2457
2495
|
if self._is_ios():
|
|
2458
|
-
return {"success": False, "
|
|
2496
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2459
2497
|
|
|
2460
2498
|
# 获取屏幕尺寸
|
|
2461
2499
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
@@ -2466,6 +2504,14 @@ class BasicMobileToolsLite:
|
|
|
2466
2504
|
import xml.etree.ElementTree as ET
|
|
2467
2505
|
root = ET.fromstring(xml_string)
|
|
2468
2506
|
|
|
2507
|
+
# 🔴 先检测是否有弹窗,避免误识别普通页面的按钮
|
|
2508
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2509
|
+
root, screen_width, screen_height
|
|
2510
|
+
)
|
|
2511
|
+
|
|
2512
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
2513
|
+
return {"success": True, "popup": False}
|
|
2514
|
+
|
|
2469
2515
|
# 关闭按钮特征
|
|
2470
2516
|
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
|
|
2471
2517
|
candidates = []
|
|
@@ -2567,27 +2613,16 @@ class BasicMobileToolsLite:
|
|
|
2567
2613
|
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2568
2614
|
best = candidates[0]
|
|
2569
2615
|
|
|
2616
|
+
# Token 优化:只返回最必要的信息
|
|
2570
2617
|
return {
|
|
2571
2618
|
"success": True,
|
|
2572
|
-
"
|
|
2573
|
-
"
|
|
2574
|
-
|
|
2575
|
-
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
2576
|
-
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2577
|
-
"bounds": best['bounds'],
|
|
2578
|
-
"size": best['size'],
|
|
2579
|
-
"score": best['score']
|
|
2580
|
-
},
|
|
2581
|
-
"click_command": f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})",
|
|
2582
|
-
"other_candidates": [
|
|
2583
|
-
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
2584
|
-
for c in candidates[1:4]
|
|
2585
|
-
] if len(candidates) > 1 else [],
|
|
2586
|
-
"screen_size": {"width": screen_width, "height": screen_height}
|
|
2619
|
+
"popup": True,
|
|
2620
|
+
"close": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2621
|
+
"cmd": f"click_by_percent({best['x_percent']},{best['y_percent']})"
|
|
2587
2622
|
}
|
|
2588
2623
|
|
|
2589
2624
|
except Exception as e:
|
|
2590
|
-
return {"success": False, "
|
|
2625
|
+
return {"success": False, "msg": str(e)}
|
|
2591
2626
|
|
|
2592
2627
|
def close_popup(self) -> Dict:
|
|
2593
2628
|
"""智能关闭弹窗(改进版)
|
|
@@ -2610,7 +2645,7 @@ class BasicMobileToolsLite:
|
|
|
2610
2645
|
|
|
2611
2646
|
# 获取屏幕尺寸
|
|
2612
2647
|
if self._is_ios():
|
|
2613
|
-
return {"success": False, "
|
|
2648
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2614
2649
|
|
|
2615
2650
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2616
2651
|
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
@@ -2630,58 +2665,18 @@ class BasicMobileToolsLite:
|
|
|
2630
2665
|
root = ET.fromstring(xml_string)
|
|
2631
2666
|
all_elements = list(root.iter())
|
|
2632
2667
|
|
|
2633
|
-
# =====
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
bounds_str = elem.attrib.get('bounds', '')
|
|
2638
|
-
class_name = elem.attrib.get('class', '')
|
|
2639
|
-
|
|
2640
|
-
if not bounds_str:
|
|
2641
|
-
continue
|
|
2642
|
-
|
|
2643
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2644
|
-
if not match:
|
|
2645
|
-
continue
|
|
2646
|
-
|
|
2647
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2648
|
-
width = x2 - x1
|
|
2649
|
-
height = y2 - y1
|
|
2650
|
-
area = width * height
|
|
2651
|
-
screen_area = screen_width * screen_height
|
|
2652
|
-
|
|
2653
|
-
# 弹窗容器特征(更严格,排除主要内容区域):
|
|
2654
|
-
# 1. 面积在屏幕的 10%-70% 之间(排除主要内容区域,通常占80%+)
|
|
2655
|
-
# 2. 宽度和高度都要小于屏幕的95%(不是全屏)
|
|
2656
|
-
# 3. 是容器类型(Layout/View/Dialog)
|
|
2657
|
-
# 4. 不在屏幕左边缘(排除从x=0开始的主要内容容器)
|
|
2658
|
-
# 5. 高度不能占据屏幕的大部分(排除主要内容区域)
|
|
2659
|
-
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Container'])
|
|
2660
|
-
area_ratio = area / screen_area
|
|
2661
|
-
is_not_fullscreen = (width < screen_width * 0.95 and height < screen_height * 0.95)
|
|
2662
|
-
is_reasonable_size = 0.10 < area_ratio < 0.70
|
|
2663
|
-
is_not_at_left_edge = x1 > screen_width * 0.05
|
|
2664
|
-
height_ratio = height / screen_height if screen_height > 0 else 0
|
|
2665
|
-
is_not_main_content = height_ratio < 0.85
|
|
2666
|
-
|
|
2667
|
-
# 排除状态栏区域(y1 通常很小)
|
|
2668
|
-
is_below_statusbar = y1 > 50
|
|
2669
|
-
|
|
2670
|
-
if is_container and is_not_fullscreen and is_reasonable_size and is_not_at_left_edge and is_not_main_content and is_below_statusbar:
|
|
2671
|
-
popup_containers.append({
|
|
2672
|
-
'bounds': (x1, y1, x2, y2),
|
|
2673
|
-
'bounds_str': bounds_str,
|
|
2674
|
-
'area': area,
|
|
2675
|
-
'area_ratio': area_ratio,
|
|
2676
|
-
'idx': idx, # 元素在 XML 中的顺序(越后越上层)
|
|
2677
|
-
'class': class_name
|
|
2678
|
-
})
|
|
2668
|
+
# ===== 第一步:使用严格的置信度检测弹窗区域 =====
|
|
2669
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2670
|
+
root, screen_width, screen_height
|
|
2671
|
+
)
|
|
2679
2672
|
|
|
2680
|
-
#
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2673
|
+
# 如果置信度不够高,记录但继续尝试查找关闭按钮
|
|
2674
|
+
popup_detected = popup_bounds is not None and popup_confidence >= 0.6
|
|
2675
|
+
|
|
2676
|
+
# 🔴 关键检查:如果没有检测到弹窗区域,直接返回"无弹窗"
|
|
2677
|
+
# 避免误点击普通页面上的"关闭"、"取消"等按钮
|
|
2678
|
+
if not popup_detected:
|
|
2679
|
+
return {"success": True, "popup": False}
|
|
2685
2680
|
|
|
2686
2681
|
# ===== 第二步:在弹窗范围内查找关闭按钮 =====
|
|
2687
2682
|
for idx, elem in enumerate(all_elements):
|
|
@@ -2813,93 +2808,16 @@ class BasicMobileToolsLite:
|
|
|
2813
2808
|
'content_desc': content_desc,
|
|
2814
2809
|
'x_percent': round(rel_x * 100, 1),
|
|
2815
2810
|
'y_percent': round(rel_y * 100, 1),
|
|
2816
|
-
'in_popup':
|
|
2811
|
+
'in_popup': popup_detected
|
|
2817
2812
|
})
|
|
2818
2813
|
|
|
2819
2814
|
except ET.ParseError:
|
|
2820
2815
|
pass
|
|
2821
2816
|
|
|
2822
2817
|
if not close_candidates:
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
popup_width = px2 - px1
|
|
2827
|
-
popup_height = py2 - py1
|
|
2828
|
-
|
|
2829
|
-
# 【优化】X按钮有三种常见位置:
|
|
2830
|
-
# 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
|
|
2831
|
-
# 2. 弹窗边界上方(浮动X按钮)
|
|
2832
|
-
# 3. 弹窗正下方(底部关闭按钮)
|
|
2833
|
-
offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
|
|
2834
|
-
offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
|
|
2835
|
-
offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
|
|
2836
|
-
|
|
2837
|
-
try_positions = [
|
|
2838
|
-
# 【最高优先级】弹窗内紧贴顶部边界
|
|
2839
|
-
(px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
|
|
2840
|
-
# 弹窗边界上方(浮动X按钮)
|
|
2841
|
-
(px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
|
|
2842
|
-
# 弹窗正下方中间(底部关闭按钮)
|
|
2843
|
-
((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
|
|
2844
|
-
# 弹窗正上方中间
|
|
2845
|
-
((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
|
|
2846
|
-
]
|
|
2847
|
-
|
|
2848
|
-
for try_x, try_y, position_name in try_positions:
|
|
2849
|
-
if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
|
|
2850
|
-
self.client.u2.click(try_x, try_y)
|
|
2851
|
-
time.sleep(0.3)
|
|
2852
|
-
|
|
2853
|
-
# 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
|
|
2854
|
-
app_check = self._check_app_switched()
|
|
2855
|
-
return_result = None
|
|
2856
|
-
|
|
2857
|
-
if app_check['switched']:
|
|
2858
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
2859
|
-
return_result = self._return_to_target_app()
|
|
2860
|
-
|
|
2861
|
-
# 尝试后截图,让 AI 判断是否成功
|
|
2862
|
-
screenshot_result = self.take_screenshot("尝试关闭后")
|
|
2863
|
-
|
|
2864
|
-
msg = f"✅ 已尝试点击常见关闭按钮位置"
|
|
2865
|
-
if app_check['switched']:
|
|
2866
|
-
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
2867
|
-
if return_result:
|
|
2868
|
-
if return_result['success']:
|
|
2869
|
-
msg += f"\n{return_result['message']}"
|
|
2870
|
-
else:
|
|
2871
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2872
|
-
|
|
2873
|
-
return {
|
|
2874
|
-
"success": True,
|
|
2875
|
-
"message": msg,
|
|
2876
|
-
"tried_positions": [p[2] for p in try_positions],
|
|
2877
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2878
|
-
"app_check": app_check,
|
|
2879
|
-
"return_to_app": return_result,
|
|
2880
|
-
"tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
|
|
2881
|
-
}
|
|
2882
|
-
|
|
2883
|
-
# 没有检测到弹窗区域,截图让 AI 分析
|
|
2884
|
-
screenshot_result = self.take_screenshot(description="页面截图", compress=True)
|
|
2885
|
-
|
|
2886
|
-
return {
|
|
2887
|
-
"success": False,
|
|
2888
|
-
"message": "❌ 未检测到弹窗区域,已截图供 AI 分析",
|
|
2889
|
-
"action_required": "请查看截图找到关闭按钮,调用 mobile_click_at_coords 点击",
|
|
2890
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2891
|
-
"screen_size": {"width": screen_width, "height": screen_height},
|
|
2892
|
-
"image_size": {
|
|
2893
|
-
"width": screenshot_result.get("image_width", screen_width),
|
|
2894
|
-
"height": screenshot_result.get("image_height", screen_height)
|
|
2895
|
-
},
|
|
2896
|
-
"original_size": {
|
|
2897
|
-
"width": screenshot_result.get("original_img_width", screen_width),
|
|
2898
|
-
"height": screenshot_result.get("original_img_height", screen_height)
|
|
2899
|
-
},
|
|
2900
|
-
"search_areas": ["弹窗右上角", "弹窗正上方", "弹窗下方中间", "屏幕右上角"],
|
|
2901
|
-
"time_warning": "⚠️ 截图分析期间弹窗可能自动消失。如果是定时弹窗,建议等待其自动消失。"
|
|
2902
|
-
}
|
|
2818
|
+
if popup_detected and popup_bounds:
|
|
2819
|
+
return {"success": False, "fallback": "vision", "popup": True}
|
|
2820
|
+
return {"success": True, "popup": False}
|
|
2903
2821
|
|
|
2904
2822
|
# 按得分排序,取最可能的
|
|
2905
2823
|
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
@@ -2917,61 +2835,22 @@ class BasicMobileToolsLite:
|
|
|
2917
2835
|
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
2918
2836
|
return_result = self._return_to_target_app()
|
|
2919
2837
|
|
|
2920
|
-
#
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
self._record_operation(
|
|
2925
|
-
'click',
|
|
2926
|
-
x=best['center_x'],
|
|
2927
|
-
y=best['center_y'],
|
|
2928
|
-
x_percent=best['x_percent'],
|
|
2929
|
-
y_percent=best['y_percent'],
|
|
2930
|
-
screen_width=screen_width,
|
|
2931
|
-
screen_height=screen_height,
|
|
2932
|
-
ref=f"close_popup_{best['position']}"
|
|
2933
|
-
)
|
|
2838
|
+
# 记录操作
|
|
2839
|
+
self._record_click('percent', f"{best['x_percent']}%,{best['y_percent']}%",
|
|
2840
|
+
best['x_percent'], best['y_percent'],
|
|
2841
|
+
element_desc=f"关闭按钮({best['position']})")
|
|
2934
2842
|
|
|
2935
|
-
#
|
|
2936
|
-
|
|
2843
|
+
# Token 优化:精简返回值
|
|
2844
|
+
result = {"success": True, "clicked": True}
|
|
2937
2845
|
if app_check['switched']:
|
|
2938
|
-
|
|
2846
|
+
result["switched"] = True
|
|
2939
2847
|
if return_result:
|
|
2940
|
-
|
|
2941
|
-
msg += f"\n{return_result['message']}"
|
|
2942
|
-
else:
|
|
2943
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2848
|
+
result["returned"] = return_result['success']
|
|
2944
2849
|
|
|
2945
|
-
|
|
2946
|
-
# 如果弹窗还在,AI 可以选择点击其他候选按钮
|
|
2947
|
-
return {
|
|
2948
|
-
"success": True,
|
|
2949
|
-
"message": msg,
|
|
2950
|
-
"clicked": {
|
|
2951
|
-
"position": best['position'],
|
|
2952
|
-
"match_type": best['match_type'],
|
|
2953
|
-
"coords": (best['center_x'], best['center_y']),
|
|
2954
|
-
"percent": (best['x_percent'], best['y_percent'])
|
|
2955
|
-
},
|
|
2956
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2957
|
-
"popup_detected": popup_bounds is not None,
|
|
2958
|
-
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
|
|
2959
|
-
"app_check": app_check,
|
|
2960
|
-
"return_to_app": return_result,
|
|
2961
|
-
"other_candidates": [
|
|
2962
|
-
{
|
|
2963
|
-
"position": c['position'],
|
|
2964
|
-
"type": c['match_type'],
|
|
2965
|
-
"coords": (c['center_x'], c['center_y']),
|
|
2966
|
-
"percent": (c['x_percent'], c['y_percent'])
|
|
2967
|
-
}
|
|
2968
|
-
for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
|
|
2969
|
-
],
|
|
2970
|
-
"tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置"
|
|
2971
|
-
}
|
|
2850
|
+
return result
|
|
2972
2851
|
|
|
2973
2852
|
except Exception as e:
|
|
2974
|
-
return {"success": False, "
|
|
2853
|
+
return {"success": False, "msg": str(e)}
|
|
2975
2854
|
|
|
2976
2855
|
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
2977
2856
|
"""根据相对坐标获取位置名称"""
|
|
@@ -3014,6 +2893,308 @@ class BasicMobileToolsLite:
|
|
|
3014
2893
|
return 0.8
|
|
3015
2894
|
else: # 中间区域
|
|
3016
2895
|
return 0.5
|
|
2896
|
+
|
|
2897
|
+
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) -> tuple:
|
|
2898
|
+
"""严格的弹窗检测 - 使用置信度评分,避免误识别普通页面
|
|
2899
|
+
|
|
2900
|
+
真正的弹窗特征:
|
|
2901
|
+
1. class 名称包含 Dialog/Popup/Alert/Modal/BottomSheet(强特征)
|
|
2902
|
+
2. resource-id 包含 dialog/popup/alert/modal(强特征)
|
|
2903
|
+
3. 有遮罩层(大面积半透明 View 在弹窗之前)
|
|
2904
|
+
4. 居中显示且非全屏
|
|
2905
|
+
5. XML 层级靠后且包含可交互元素
|
|
2906
|
+
|
|
2907
|
+
Returns:
|
|
2908
|
+
(popup_bounds, confidence) 或 (None, 0)
|
|
2909
|
+
confidence >= 0.6 才认为是弹窗
|
|
2910
|
+
"""
|
|
2911
|
+
import re
|
|
2912
|
+
|
|
2913
|
+
screen_area = screen_width * screen_height
|
|
2914
|
+
|
|
2915
|
+
# 收集所有元素信息
|
|
2916
|
+
all_elements = []
|
|
2917
|
+
for idx, elem in enumerate(root.iter()):
|
|
2918
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2919
|
+
if not bounds_str:
|
|
2920
|
+
continue
|
|
2921
|
+
|
|
2922
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2923
|
+
if not match:
|
|
2924
|
+
continue
|
|
2925
|
+
|
|
2926
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2927
|
+
width = x2 - x1
|
|
2928
|
+
height = y2 - y1
|
|
2929
|
+
area = width * height
|
|
2930
|
+
|
|
2931
|
+
class_name = elem.attrib.get('class', '')
|
|
2932
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
2933
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2934
|
+
|
|
2935
|
+
all_elements.append({
|
|
2936
|
+
'idx': idx,
|
|
2937
|
+
'bounds': (x1, y1, x2, y2),
|
|
2938
|
+
'width': width,
|
|
2939
|
+
'height': height,
|
|
2940
|
+
'area': area,
|
|
2941
|
+
'area_ratio': area / screen_area if screen_area > 0 else 0,
|
|
2942
|
+
'class': class_name,
|
|
2943
|
+
'resource_id': resource_id,
|
|
2944
|
+
'clickable': clickable,
|
|
2945
|
+
'center_x': (x1 + x2) // 2,
|
|
2946
|
+
'center_y': (y1 + y2) // 2,
|
|
2947
|
+
})
|
|
2948
|
+
|
|
2949
|
+
if not all_elements:
|
|
2950
|
+
return None, 0
|
|
2951
|
+
|
|
2952
|
+
# 弹窗检测关键词
|
|
2953
|
+
dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
|
|
2954
|
+
dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
|
|
2955
|
+
|
|
2956
|
+
popup_candidates = []
|
|
2957
|
+
has_mask_layer = False
|
|
2958
|
+
mask_idx = -1
|
|
2959
|
+
|
|
2960
|
+
for elem in all_elements:
|
|
2961
|
+
x1, y1, x2, y2 = elem['bounds']
|
|
2962
|
+
class_name = elem['class']
|
|
2963
|
+
resource_id = elem['resource_id']
|
|
2964
|
+
area_ratio = elem['area_ratio']
|
|
2965
|
+
|
|
2966
|
+
# 检测遮罩层(大面积、几乎全屏、通常是 FrameLayout/View)
|
|
2967
|
+
if area_ratio > 0.85 and elem['width'] >= screen_width * 0.95:
|
|
2968
|
+
# 可能是遮罩层,记录位置
|
|
2969
|
+
if 'FrameLayout' in class_name or 'View' in class_name:
|
|
2970
|
+
has_mask_layer = True
|
|
2971
|
+
mask_idx = elem['idx']
|
|
2972
|
+
|
|
2973
|
+
# 跳过全屏元素
|
|
2974
|
+
if area_ratio > 0.9:
|
|
2975
|
+
continue
|
|
2976
|
+
|
|
2977
|
+
# 跳过太小的元素
|
|
2978
|
+
if area_ratio < 0.05:
|
|
2979
|
+
continue
|
|
2980
|
+
|
|
2981
|
+
# 跳过状态栏区域
|
|
2982
|
+
if y1 < 50:
|
|
2983
|
+
continue
|
|
2984
|
+
|
|
2985
|
+
confidence = 0.0
|
|
2986
|
+
|
|
2987
|
+
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
2988
|
+
if any(kw in class_name for kw in dialog_class_keywords):
|
|
2989
|
+
confidence += 0.5
|
|
2990
|
+
|
|
2991
|
+
# 【强特征】resource-id 包含弹窗关键词 (+0.4)
|
|
2992
|
+
if any(kw in resource_id.lower() for kw in dialog_id_keywords):
|
|
2993
|
+
confidence += 0.4
|
|
2994
|
+
|
|
2995
|
+
# 【中等特征】居中显示 (+0.2)
|
|
2996
|
+
center_x = elem['center_x']
|
|
2997
|
+
center_y = elem['center_y']
|
|
2998
|
+
is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
|
|
2999
|
+
is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
|
|
3000
|
+
if is_centered_x and is_centered_y:
|
|
3001
|
+
confidence += 0.2
|
|
3002
|
+
elif is_centered_x:
|
|
3003
|
+
confidence += 0.1
|
|
3004
|
+
|
|
3005
|
+
# 【中等特征】非全屏但有一定大小 (+0.15)
|
|
3006
|
+
if 0.15 < area_ratio < 0.75:
|
|
3007
|
+
confidence += 0.15
|
|
3008
|
+
|
|
3009
|
+
# 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
|
|
3010
|
+
if elem['idx'] > len(all_elements) * 0.5:
|
|
3011
|
+
confidence += 0.1
|
|
3012
|
+
|
|
3013
|
+
# 【弱特征】有遮罩层且在遮罩层之后 (+0.15)
|
|
3014
|
+
if has_mask_layer and elem['idx'] > mask_idx:
|
|
3015
|
+
confidence += 0.15
|
|
3016
|
+
|
|
3017
|
+
# 只有达到阈值才加入候选
|
|
3018
|
+
if confidence >= 0.3:
|
|
3019
|
+
popup_candidates.append({
|
|
3020
|
+
'bounds': elem['bounds'],
|
|
3021
|
+
'confidence': confidence,
|
|
3022
|
+
'class': class_name,
|
|
3023
|
+
'resource_id': resource_id,
|
|
3024
|
+
'idx': elem['idx']
|
|
3025
|
+
})
|
|
3026
|
+
|
|
3027
|
+
if not popup_candidates:
|
|
3028
|
+
return None, 0
|
|
3029
|
+
|
|
3030
|
+
# 选择置信度最高的
|
|
3031
|
+
popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
|
|
3032
|
+
best = popup_candidates[0]
|
|
3033
|
+
|
|
3034
|
+
# 只有置信度 >= 0.6 才返回弹窗
|
|
3035
|
+
if best['confidence'] >= 0.6:
|
|
3036
|
+
return best['bounds'], best['confidence']
|
|
3037
|
+
|
|
3038
|
+
return None, best['confidence']
|
|
3039
|
+
|
|
3040
|
+
def start_toast_watch(self) -> Dict:
|
|
3041
|
+
"""开始监听 Toast(仅 Android)
|
|
3042
|
+
|
|
3043
|
+
⚠️ 必须在执行操作之前调用!
|
|
3044
|
+
|
|
3045
|
+
正确流程:
|
|
3046
|
+
1. 调用 mobile_start_toast_watch() 开始监听
|
|
3047
|
+
2. 执行操作(如点击提交按钮)
|
|
3048
|
+
3. 调用 mobile_get_toast() 获取 Toast 内容
|
|
3049
|
+
|
|
3050
|
+
Returns:
|
|
3051
|
+
监听状态
|
|
3052
|
+
"""
|
|
3053
|
+
if self._is_ios():
|
|
3054
|
+
return {
|
|
3055
|
+
"success": False,
|
|
3056
|
+
"message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
|
|
3057
|
+
}
|
|
3058
|
+
|
|
3059
|
+
try:
|
|
3060
|
+
# 清除缓存并开始监听
|
|
3061
|
+
self.client.u2.toast.reset()
|
|
3062
|
+
return {
|
|
3063
|
+
"success": True,
|
|
3064
|
+
"message": "✅ Toast 监听已开启,请立即执行操作,然后调用 mobile_get_toast 获取结果"
|
|
3065
|
+
}
|
|
3066
|
+
except Exception as e:
|
|
3067
|
+
return {
|
|
3068
|
+
"success": False,
|
|
3069
|
+
"message": f"❌ 开启 Toast 监听失败: {e}"
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3072
|
+
def get_toast(self, timeout: float = 5.0, reset_first: bool = False) -> Dict:
|
|
3073
|
+
"""获取 Toast 消息(仅 Android)
|
|
3074
|
+
|
|
3075
|
+
Toast 是 Android 系统级的短暂提示消息,常用于显示操作结果。
|
|
3076
|
+
|
|
3077
|
+
⚠️ 推荐用法(两步走):
|
|
3078
|
+
1. 先调用 mobile_start_toast_watch() 开始监听
|
|
3079
|
+
2. 执行操作(如点击提交按钮)
|
|
3080
|
+
3. 调用 mobile_get_toast() 获取 Toast
|
|
3081
|
+
|
|
3082
|
+
或者设置 reset_first=True,会自动 reset 后等待(适合操作已自动触发的场景)
|
|
3083
|
+
|
|
3084
|
+
Args:
|
|
3085
|
+
timeout: 等待 Toast 出现的超时时间(秒),默认 5 秒
|
|
3086
|
+
reset_first: 是否先 reset(清除旧缓存),默认 False
|
|
3087
|
+
|
|
3088
|
+
Returns:
|
|
3089
|
+
包含 Toast 消息的字典
|
|
3090
|
+
"""
|
|
3091
|
+
if self._is_ios():
|
|
3092
|
+
return {
|
|
3093
|
+
"success": False,
|
|
3094
|
+
"message": "❌ iOS 不支持 Toast 检测,Toast 是 Android 特有功能"
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
try:
|
|
3098
|
+
if reset_first:
|
|
3099
|
+
# 清除旧缓存,适合等待即将出现的 Toast
|
|
3100
|
+
self.client.u2.toast.reset()
|
|
3101
|
+
|
|
3102
|
+
# 等待并获取 Toast 消息
|
|
3103
|
+
toast_message = self.client.u2.toast.get_message(
|
|
3104
|
+
wait_timeout=timeout,
|
|
3105
|
+
default=None
|
|
3106
|
+
)
|
|
3107
|
+
|
|
3108
|
+
if toast_message:
|
|
3109
|
+
return {
|
|
3110
|
+
"success": True,
|
|
3111
|
+
"toast_found": True,
|
|
3112
|
+
"message": toast_message,
|
|
3113
|
+
"tip": "Toast 消息获取成功"
|
|
3114
|
+
}
|
|
3115
|
+
else:
|
|
3116
|
+
return {
|
|
3117
|
+
"success": True,
|
|
3118
|
+
"toast_found": False,
|
|
3119
|
+
"message": None,
|
|
3120
|
+
"tip": f"在 {timeout} 秒内未检测到 Toast。提示:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具"
|
|
3121
|
+
}
|
|
3122
|
+
except Exception as e:
|
|
3123
|
+
return {
|
|
3124
|
+
"success": False,
|
|
3125
|
+
"message": f"❌ 获取 Toast 失败: {e}"
|
|
3126
|
+
}
|
|
3127
|
+
|
|
3128
|
+
def assert_toast(self, expected_text: str, timeout: float = 5.0, contains: bool = True) -> Dict:
|
|
3129
|
+
"""断言 Toast 消息(仅 Android)
|
|
3130
|
+
|
|
3131
|
+
等待 Toast 出现并验证内容是否符合预期。
|
|
3132
|
+
|
|
3133
|
+
⚠️ 推荐用法:先调用 mobile_start_toast_watch,再执行操作,最后调用此工具
|
|
3134
|
+
|
|
3135
|
+
Args:
|
|
3136
|
+
expected_text: 期望的 Toast 文本
|
|
3137
|
+
timeout: 等待超时时间(秒)
|
|
3138
|
+
contains: True 表示包含匹配,False 表示精确匹配
|
|
3139
|
+
|
|
3140
|
+
Returns:
|
|
3141
|
+
断言结果
|
|
3142
|
+
"""
|
|
3143
|
+
if self._is_ios():
|
|
3144
|
+
return {
|
|
3145
|
+
"success": False,
|
|
3146
|
+
"passed": False,
|
|
3147
|
+
"message": "❌ iOS 不支持 Toast 检测"
|
|
3148
|
+
}
|
|
3149
|
+
|
|
3150
|
+
try:
|
|
3151
|
+
# 获取 Toast(不 reset,假设之前已经调用过 start_toast_watch)
|
|
3152
|
+
toast_message = self.client.u2.toast.get_message(
|
|
3153
|
+
wait_timeout=timeout,
|
|
3154
|
+
default=None
|
|
3155
|
+
)
|
|
3156
|
+
|
|
3157
|
+
if toast_message is None:
|
|
3158
|
+
return {
|
|
3159
|
+
"success": True,
|
|
3160
|
+
"passed": False,
|
|
3161
|
+
"expected": expected_text,
|
|
3162
|
+
"actual": None,
|
|
3163
|
+
"message": f"❌ 断言失败:未检测到 Toast 消息"
|
|
3164
|
+
}
|
|
3165
|
+
|
|
3166
|
+
# 匹配检查
|
|
3167
|
+
if contains:
|
|
3168
|
+
passed = expected_text in toast_message
|
|
3169
|
+
match_type = "包含"
|
|
3170
|
+
else:
|
|
3171
|
+
passed = expected_text == toast_message
|
|
3172
|
+
match_type = "精确"
|
|
3173
|
+
|
|
3174
|
+
if passed:
|
|
3175
|
+
return {
|
|
3176
|
+
"success": True,
|
|
3177
|
+
"passed": True,
|
|
3178
|
+
"expected": expected_text,
|
|
3179
|
+
"actual": toast_message,
|
|
3180
|
+
"match_type": match_type,
|
|
3181
|
+
"message": f"✅ Toast 断言通过:'{toast_message}'"
|
|
3182
|
+
}
|
|
3183
|
+
else:
|
|
3184
|
+
return {
|
|
3185
|
+
"success": True,
|
|
3186
|
+
"passed": False,
|
|
3187
|
+
"expected": expected_text,
|
|
3188
|
+
"actual": toast_message,
|
|
3189
|
+
"match_type": match_type,
|
|
3190
|
+
"message": f"❌ Toast 断言失败:期望 '{expected_text}',实际 '{toast_message}'"
|
|
3191
|
+
}
|
|
3192
|
+
except Exception as e:
|
|
3193
|
+
return {
|
|
3194
|
+
"success": False,
|
|
3195
|
+
"passed": False,
|
|
3196
|
+
"message": f"❌ Toast 断言异常: {e}"
|
|
3197
|
+
}
|
|
3017
3198
|
|
|
3018
3199
|
def assert_text(self, text: str) -> Dict:
|
|
3019
3200
|
"""检查页面是否包含文本(支持精确匹配和包含匹配)"""
|
|
@@ -3102,8 +3283,13 @@ class BasicMobileToolsLite:
|
|
|
3102
3283
|
"1. 文本定位 - 最稳定,跨设备兼容",
|
|
3103
3284
|
"2. ID 定位 - 稳定,跨设备兼容",
|
|
3104
3285
|
"3. 百分比定位 - 跨分辨率兼容(坐标自动转换)",
|
|
3286
|
+
"",
|
|
3287
|
+
"运行方式:",
|
|
3288
|
+
" pytest {filename} -v # 使用 pytest 运行",
|
|
3289
|
+
" python {filename} # 直接运行",
|
|
3105
3290
|
f'"""',
|
|
3106
3291
|
"import time",
|
|
3292
|
+
"import pytest",
|
|
3107
3293
|
"import uiautomator2 as u2",
|
|
3108
3294
|
"",
|
|
3109
3295
|
f'PACKAGE_NAME = "{package_name}"',
|
|
@@ -3179,22 +3365,52 @@ class BasicMobileToolsLite:
|
|
|
3179
3365
|
" return True",
|
|
3180
3366
|
"",
|
|
3181
3367
|
"",
|
|
3182
|
-
"def
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3368
|
+
"def swipe_direction(d, direction):",
|
|
3369
|
+
' """',
|
|
3370
|
+
' 通用滑动方法(兼容所有 uiautomator2 版本)',
|
|
3371
|
+
' ',
|
|
3372
|
+
' Args:',
|
|
3373
|
+
' d: uiautomator2 设备对象',
|
|
3374
|
+
' direction: 滑动方向 (up/down/left/right)',
|
|
3375
|
+
' """',
|
|
3376
|
+
" info = d.info",
|
|
3377
|
+
" width = info.get('displayWidth', 0)",
|
|
3378
|
+
" height = info.get('displayHeight', 0)",
|
|
3379
|
+
" cx, cy = width // 2, height // 2",
|
|
3190
3380
|
" ",
|
|
3191
|
-
"
|
|
3381
|
+
" if direction == 'up':",
|
|
3382
|
+
" d.swipe(cx, int(height * 0.8), cx, int(height * 0.3))",
|
|
3383
|
+
" elif direction == 'down':",
|
|
3384
|
+
" d.swipe(cx, int(height * 0.3), cx, int(height * 0.8))",
|
|
3385
|
+
" elif direction == 'left':",
|
|
3386
|
+
" d.swipe(int(width * 0.8), cy, int(width * 0.2), cy)",
|
|
3387
|
+
" elif direction == 'right':",
|
|
3388
|
+
" d.swipe(int(width * 0.2), cy, int(width * 0.8), cy)",
|
|
3389
|
+
" return True",
|
|
3390
|
+
"",
|
|
3391
|
+
"",
|
|
3392
|
+
"# ========== pytest fixture ==========",
|
|
3393
|
+
"@pytest.fixture(scope='function')",
|
|
3394
|
+
"def device():",
|
|
3395
|
+
' """pytest fixture: 连接设备并启动应用"""',
|
|
3396
|
+
" d = u2.connect()",
|
|
3397
|
+
" d.implicitly_wait(10)",
|
|
3398
|
+
" d.app_start(PACKAGE_NAME)",
|
|
3399
|
+
" time.sleep(LAUNCH_WAIT)",
|
|
3192
3400
|
" if CLOSE_AD_ON_LAUNCH:",
|
|
3193
3401
|
" close_ad_if_exists(d)",
|
|
3402
|
+
" yield d",
|
|
3403
|
+
" # 测试结束后可选择关闭应用",
|
|
3404
|
+
" # d.app_stop(PACKAGE_NAME)",
|
|
3405
|
+
"",
|
|
3406
|
+
"",
|
|
3407
|
+
f"def test_{safe_name}(device):",
|
|
3408
|
+
' """测试用例主函数"""',
|
|
3409
|
+
" d = device",
|
|
3194
3410
|
" ",
|
|
3195
3411
|
]
|
|
3196
3412
|
|
|
3197
|
-
#
|
|
3413
|
+
# 生成操作代码(使用标准记录格式,逻辑更简洁)
|
|
3198
3414
|
step_num = 0
|
|
3199
3415
|
for op in self.operation_history:
|
|
3200
3416
|
action = op.get('action')
|
|
@@ -3206,131 +3422,122 @@ class BasicMobileToolsLite:
|
|
|
3206
3422
|
step_num += 1
|
|
3207
3423
|
|
|
3208
3424
|
if action == 'click':
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
script_lines.append(f"
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
script_lines.append(f"
|
|
3239
|
-
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
3240
|
-
elif has_coords:
|
|
3241
|
-
# 4️⃣ 坐标兜底(不推荐,仅用于无法获取百分比的情况)
|
|
3242
|
-
desc = f" ({element})" if element else ""
|
|
3243
|
-
script_lines.append(f" # 步骤{step_num}: 点击坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
3244
|
-
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
3425
|
+
# 新格式:使用 locator_type 和 locator_value
|
|
3426
|
+
locator_type = op.get('locator_type', '')
|
|
3427
|
+
locator_value = op.get('locator_value', '')
|
|
3428
|
+
locator_attr = op.get('locator_attr', 'text')
|
|
3429
|
+
element_desc = op.get('element_desc', '')
|
|
3430
|
+
x_pct = op.get('x_percent', 0)
|
|
3431
|
+
y_pct = op.get('y_percent', 0)
|
|
3432
|
+
|
|
3433
|
+
# 转义单引号
|
|
3434
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3435
|
+
|
|
3436
|
+
if locator_type == 'text':
|
|
3437
|
+
# 文本定位(最稳定)
|
|
3438
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (文本定位)")
|
|
3439
|
+
if locator_attr == 'description':
|
|
3440
|
+
script_lines.append(f" safe_click(d, d(description='{value_escaped}'))")
|
|
3441
|
+
elif locator_attr == 'descriptionContains':
|
|
3442
|
+
script_lines.append(f" safe_click(d, d(descriptionContains='{value_escaped}'))")
|
|
3443
|
+
elif locator_attr == 'textContains':
|
|
3444
|
+
script_lines.append(f" safe_click(d, d(textContains='{value_escaped}'))")
|
|
3445
|
+
else:
|
|
3446
|
+
script_lines.append(f" safe_click(d, d(text='{value_escaped}'))")
|
|
3447
|
+
elif locator_type == 'id':
|
|
3448
|
+
# ID 定位(稳定)
|
|
3449
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (ID定位)")
|
|
3450
|
+
script_lines.append(f" safe_click(d, d(resourceId='{value_escaped}'))")
|
|
3451
|
+
elif locator_type == 'percent':
|
|
3452
|
+
# 百分比定位(跨分辨率兼容)
|
|
3453
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{element_desc}' (百分比定位)")
|
|
3454
|
+
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
|
|
3245
3455
|
else:
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3456
|
+
# 兼容旧格式
|
|
3457
|
+
ref = op.get('ref', '')
|
|
3458
|
+
if ref:
|
|
3459
|
+
ref_escaped = ref.replace("'", "\\'")
|
|
3460
|
+
script_lines.append(f" # 步骤{step_num}: 点击 '{ref}'")
|
|
3461
|
+
script_lines.append(f" safe_click(d, d(text='{ref_escaped}'))")
|
|
3462
|
+
else:
|
|
3463
|
+
continue
|
|
3464
|
+
|
|
3465
|
+
script_lines.append(" time.sleep(0.5)")
|
|
3249
3466
|
script_lines.append(" ")
|
|
3250
3467
|
|
|
3251
3468
|
elif action == 'input':
|
|
3252
3469
|
text = op.get('text', '')
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
if
|
|
3262
|
-
#
|
|
3263
|
-
script_lines.append(f"
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
script_lines.append(f"
|
|
3269
|
-
elif has_percent:
|
|
3270
|
-
x_pct = op['x_percent']
|
|
3271
|
-
y_pct = op['y_percent']
|
|
3272
|
-
script_lines.append(f" # 步骤{step_num}: 点击后输入 (百分比定位)")
|
|
3470
|
+
locator_type = op.get('locator_type', '')
|
|
3471
|
+
locator_value = op.get('locator_value', '')
|
|
3472
|
+
x_pct = op.get('x_percent', 0)
|
|
3473
|
+
y_pct = op.get('y_percent', 0)
|
|
3474
|
+
|
|
3475
|
+
text_escaped = text.replace("'", "\\'")
|
|
3476
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3477
|
+
|
|
3478
|
+
if locator_type == 'id':
|
|
3479
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (ID定位)")
|
|
3480
|
+
script_lines.append(f" d(resourceId='{value_escaped}').set_text('{text_escaped}')")
|
|
3481
|
+
elif locator_type == 'class':
|
|
3482
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}' (类名定位)")
|
|
3483
|
+
script_lines.append(f" d(className='android.widget.EditText').set_text('{text_escaped}')")
|
|
3484
|
+
elif x_pct > 0 and y_pct > 0:
|
|
3485
|
+
script_lines.append(f" # 步骤{step_num}: 点击后输入 '{text}'")
|
|
3273
3486
|
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
|
|
3274
|
-
script_lines.append(
|
|
3275
|
-
script_lines.append(f" d.send_keys('{
|
|
3276
|
-
elif has_coords:
|
|
3277
|
-
script_lines.append(f" # 步骤{step_num}: 点击坐标后输入 (⚠️ 可能不兼容其他分辨率)")
|
|
3278
|
-
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
3279
|
-
script_lines.append(f" time.sleep(0.3)")
|
|
3280
|
-
script_lines.append(f" d.send_keys('{text}')")
|
|
3487
|
+
script_lines.append(" time.sleep(0.3)")
|
|
3488
|
+
script_lines.append(f" d.send_keys('{text_escaped}')")
|
|
3281
3489
|
else:
|
|
3282
|
-
#
|
|
3283
|
-
|
|
3490
|
+
# 兼容旧格式
|
|
3491
|
+
ref = op.get('ref', '')
|
|
3492
|
+
if ref:
|
|
3493
|
+
script_lines.append(f" # 步骤{step_num}: 输入 '{text}'")
|
|
3494
|
+
script_lines.append(f" d(resourceId='{ref}').set_text('{text_escaped}')")
|
|
3495
|
+
else:
|
|
3496
|
+
continue
|
|
3497
|
+
|
|
3284
3498
|
script_lines.append(" time.sleep(0.5)")
|
|
3285
3499
|
script_lines.append(" ")
|
|
3286
3500
|
|
|
3287
3501
|
elif action == 'long_press':
|
|
3288
|
-
|
|
3289
|
-
|
|
3502
|
+
locator_type = op.get('locator_type', '')
|
|
3503
|
+
locator_value = op.get('locator_value', '')
|
|
3504
|
+
locator_attr = op.get('locator_attr', 'text')
|
|
3505
|
+
element_desc = op.get('element_desc', '')
|
|
3290
3506
|
duration = op.get('duration', 1.0)
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
script_lines.append(f" # 步骤{step_num}:
|
|
3307
|
-
script_lines.append(f" d
|
|
3308
|
-
elif ref and (':id/' in ref or ref.startswith('com.')):
|
|
3309
|
-
# 2️⃣ 使用 resource-id(稳定)
|
|
3310
|
-
script_lines.append(f" # 步骤{step_num}: 长按元素 (ID定位)")
|
|
3311
|
-
script_lines.append(f" d(resourceId='{ref}').long_click(duration={duration})")
|
|
3312
|
-
elif has_percent:
|
|
3313
|
-
# 使用百分比
|
|
3314
|
-
x_pct = op['x_percent']
|
|
3315
|
-
y_pct = op['y_percent']
|
|
3316
|
-
desc = f" ({element})" if element else ""
|
|
3317
|
-
script_lines.append(f" # 步骤{step_num}: 长按位置{desc} (百分比定位,跨分辨率兼容)")
|
|
3318
|
-
script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
3319
|
-
elif has_coords:
|
|
3320
|
-
# 坐标兜底
|
|
3321
|
-
desc = f" ({element})" if element else ""
|
|
3322
|
-
script_lines.append(f" # 步骤{step_num}: 长按坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
3323
|
-
script_lines.append(f" d.long_click({op['x']}, {op['y']}, duration={duration})")
|
|
3507
|
+
x_pct = op.get('x_percent', 0)
|
|
3508
|
+
y_pct = op.get('y_percent', 0)
|
|
3509
|
+
|
|
3510
|
+
value_escaped = locator_value.replace("'", "\\'") if locator_value else ''
|
|
3511
|
+
|
|
3512
|
+
if locator_type == 'text':
|
|
3513
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3514
|
+
if locator_attr == 'description':
|
|
3515
|
+
script_lines.append(f" d(description='{value_escaped}').long_click(duration={duration})")
|
|
3516
|
+
else:
|
|
3517
|
+
script_lines.append(f" d(text='{value_escaped}').long_click(duration={duration})")
|
|
3518
|
+
elif locator_type == 'id':
|
|
3519
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3520
|
+
script_lines.append(f" d(resourceId='{value_escaped}').long_click(duration={duration})")
|
|
3521
|
+
elif locator_type == 'percent':
|
|
3522
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{element_desc}'")
|
|
3523
|
+
script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration})")
|
|
3324
3524
|
else:
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3525
|
+
# 兼容旧格式
|
|
3526
|
+
ref = op.get('ref', '')
|
|
3527
|
+
if ref:
|
|
3528
|
+
ref_escaped = ref.replace("'", "\\'")
|
|
3529
|
+
script_lines.append(f" # 步骤{step_num}: 长按 '{ref}'")
|
|
3530
|
+
script_lines.append(f" d(text='{ref_escaped}').long_click(duration={duration})")
|
|
3531
|
+
else:
|
|
3532
|
+
continue
|
|
3533
|
+
|
|
3534
|
+
script_lines.append(" time.sleep(0.5)")
|
|
3328
3535
|
script_lines.append(" ")
|
|
3329
3536
|
|
|
3330
3537
|
elif action == 'swipe':
|
|
3331
3538
|
direction = op.get('direction', 'up')
|
|
3332
3539
|
script_lines.append(f" # 步骤{step_num}: 滑动 {direction}")
|
|
3333
|
-
script_lines.append(f" d
|
|
3540
|
+
script_lines.append(f" swipe_direction(d, '{direction}')")
|
|
3334
3541
|
script_lines.append(" time.sleep(0.5)")
|
|
3335
3542
|
script_lines.append(" ")
|
|
3336
3543
|
|
|
@@ -3345,8 +3552,16 @@ class BasicMobileToolsLite:
|
|
|
3345
3552
|
" print('✅ 测试完成')",
|
|
3346
3553
|
"",
|
|
3347
3554
|
"",
|
|
3555
|
+
"# ========== 直接运行入口 ==========",
|
|
3348
3556
|
"if __name__ == '__main__':",
|
|
3349
|
-
"
|
|
3557
|
+
" # 直接运行时,手动创建设备连接",
|
|
3558
|
+
" _d = u2.connect()",
|
|
3559
|
+
" _d.implicitly_wait(10)",
|
|
3560
|
+
" _d.app_start(PACKAGE_NAME)",
|
|
3561
|
+
" time.sleep(LAUNCH_WAIT)",
|
|
3562
|
+
" if CLOSE_AD_ON_LAUNCH:",
|
|
3563
|
+
" close_ad_if_exists(_d)",
|
|
3564
|
+
f" test_{safe_name}(_d)",
|
|
3350
3565
|
])
|
|
3351
3566
|
|
|
3352
3567
|
script = '\n'.join(script_lines)
|
|
@@ -3355,8 +3570,11 @@ class BasicMobileToolsLite:
|
|
|
3355
3570
|
output_dir = Path("tests")
|
|
3356
3571
|
output_dir.mkdir(exist_ok=True)
|
|
3357
3572
|
|
|
3573
|
+
# 确保文件名符合 pytest 规范(以 test_ 开头)
|
|
3358
3574
|
if not filename.endswith('.py'):
|
|
3359
3575
|
filename = f"{filename}.py"
|
|
3576
|
+
if not filename.startswith('test_'):
|
|
3577
|
+
filename = f"test_{filename}"
|
|
3360
3578
|
|
|
3361
3579
|
file_path = output_dir / filename
|
|
3362
3580
|
file_path.write_text(script, encoding='utf-8')
|
|
@@ -3364,7 +3582,7 @@ class BasicMobileToolsLite:
|
|
|
3364
3582
|
return {
|
|
3365
3583
|
"success": True,
|
|
3366
3584
|
"file_path": str(file_path),
|
|
3367
|
-
"message": f"✅ 脚本已生成: {file_path}",
|
|
3585
|
+
"message": f"✅ 脚本已生成: {file_path}\n💡 运行方式: pytest {file_path} -v 或 python {file_path}",
|
|
3368
3586
|
"operations_count": len(self.operation_history),
|
|
3369
3587
|
"preview": script[:500] + "..."
|
|
3370
3588
|
}
|
|
@@ -3533,10 +3751,28 @@ class BasicMobileToolsLite:
|
|
|
3533
3751
|
try:
|
|
3534
3752
|
import xml.etree.ElementTree as ET
|
|
3535
3753
|
|
|
3536
|
-
# ========== 第
|
|
3754
|
+
# ========== 第0步:先检测是否有弹窗 ==========
|
|
3537
3755
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
3538
3756
|
root = ET.fromstring(xml_string)
|
|
3539
3757
|
|
|
3758
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3759
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
3760
|
+
|
|
3761
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3762
|
+
root, screen_width, screen_height
|
|
3763
|
+
)
|
|
3764
|
+
|
|
3765
|
+
# 如果没有检测到弹窗,直接返回"无弹窗"
|
|
3766
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
3767
|
+
result["success"] = True
|
|
3768
|
+
result["method"] = None
|
|
3769
|
+
result["message"] = "ℹ️ 当前页面未检测到弹窗,无需关闭"
|
|
3770
|
+
result["popup_detected"] = False
|
|
3771
|
+
result["popup_confidence"] = popup_confidence
|
|
3772
|
+
return result
|
|
3773
|
+
|
|
3774
|
+
# ========== 第1步:控件树查找关闭按钮 ==========
|
|
3775
|
+
|
|
3540
3776
|
# 关闭按钮的常见特征
|
|
3541
3777
|
close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
|
|
3542
3778
|
close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
|
|
@@ -3615,12 +3851,6 @@ class BasicMobileToolsLite:
|
|
|
3615
3851
|
cx, cy = best['center']
|
|
3616
3852
|
bounds = best['bounds']
|
|
3617
3853
|
|
|
3618
|
-
# 点击前截图(用于自动学习)
|
|
3619
|
-
pre_screenshot = None
|
|
3620
|
-
if auto_learn:
|
|
3621
|
-
pre_result = self.take_screenshot(description="关闭前", compress=False)
|
|
3622
|
-
pre_screenshot = pre_result.get("screenshot_path")
|
|
3623
|
-
|
|
3624
3854
|
# 点击(click_at_coords 内部已包含应用状态检查和自动返回)
|
|
3625
3855
|
click_result = self.click_at_coords(cx, cy)
|
|
3626
3856
|
time.sleep(0.5)
|
|
@@ -3650,17 +3880,11 @@ class BasicMobileToolsLite:
|
|
|
3650
3880
|
result["message"] = msg
|
|
3651
3881
|
result["app_check"] = app_check
|
|
3652
3882
|
result["return_to_app"] = return_result
|
|
3653
|
-
|
|
3654
|
-
# 自动学习:检查这个 X 是否已在模板库,不在就添加
|
|
3655
|
-
if auto_learn and pre_screenshot:
|
|
3656
|
-
learn_result = self._auto_learn_template(pre_screenshot, bounds)
|
|
3657
|
-
if learn_result:
|
|
3658
|
-
result["learned_template"] = learn_result
|
|
3659
|
-
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
3883
|
+
result["tip"] = "💡 建议调用 mobile_screenshot_with_som 确认弹窗是否已关闭"
|
|
3660
3884
|
|
|
3661
3885
|
return result
|
|
3662
3886
|
|
|
3663
|
-
# ========== 第2
|
|
3887
|
+
# ========== 第2步:模板匹配(自动执行,不需要 AI 介入)==========
|
|
3664
3888
|
screenshot_path = None
|
|
3665
3889
|
try:
|
|
3666
3890
|
from .template_matcher import TemplateMatcher
|
|
@@ -3679,16 +3903,14 @@ class BasicMobileToolsLite:
|
|
|
3679
3903
|
x_pct = best["percent"]["x"]
|
|
3680
3904
|
y_pct = best["percent"]["y"]
|
|
3681
3905
|
|
|
3682
|
-
#
|
|
3906
|
+
# 点击
|
|
3683
3907
|
click_result = self.click_by_percent(x_pct, y_pct)
|
|
3684
3908
|
time.sleep(0.5)
|
|
3685
3909
|
|
|
3686
|
-
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
3687
3910
|
app_check = self._check_app_switched()
|
|
3688
3911
|
return_result = None
|
|
3689
3912
|
|
|
3690
3913
|
if app_check['switched']:
|
|
3691
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
3692
3914
|
return_result = self._return_to_target_app()
|
|
3693
3915
|
|
|
3694
3916
|
result["success"] = True
|
|
@@ -3699,12 +3921,9 @@ class BasicMobileToolsLite:
|
|
|
3699
3921
|
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
3700
3922
|
|
|
3701
3923
|
if app_check['switched']:
|
|
3702
|
-
msg += f"\n⚠️
|
|
3924
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
3703
3925
|
if return_result:
|
|
3704
|
-
|
|
3705
|
-
msg += f"\n{return_result['message']}"
|
|
3706
|
-
else:
|
|
3707
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
3926
|
+
msg += f"\n{return_result['message']}"
|
|
3708
3927
|
|
|
3709
3928
|
result["message"] = msg
|
|
3710
3929
|
result["app_check"] = app_check
|
|
@@ -3716,17 +3935,12 @@ class BasicMobileToolsLite:
|
|
|
3716
3935
|
except Exception:
|
|
3717
3936
|
pass # 模板匹配失败,继续下一步
|
|
3718
3937
|
|
|
3719
|
-
# ========== 第3
|
|
3720
|
-
if not screenshot_path:
|
|
3721
|
-
screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
|
|
3722
|
-
|
|
3938
|
+
# ========== 第3步:控件树和模板匹配都失败,提示 AI 使用视觉识别 ==========
|
|
3723
3939
|
result["success"] = False
|
|
3940
|
+
result["fallback"] = "vision"
|
|
3724
3941
|
result["method"] = None
|
|
3725
|
-
result["
|
|
3726
|
-
|
|
3727
|
-
"💡 找到后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
3728
|
-
result["screenshot"] = screenshot_result if not screenshot_path else {"screenshot_path": screenshot_path}
|
|
3729
|
-
result["need_ai_analysis"] = True
|
|
3942
|
+
result["popup_detected"] = True
|
|
3943
|
+
result["message"] = "⚠️ 控件树和模板匹配都未找到关闭按钮,请调用 mobile_screenshot_with_som 截图后用 click_by_som 点击"
|
|
3730
3944
|
|
|
3731
3945
|
return result
|
|
3732
3946
|
|