mobile-mcp-ai 2.6.7__py3-none-any.whl → 2.6.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/core/basic_tools_lite.py +1501 -669
- mobile_mcp/mcp_tools/mcp_server.py +153 -45
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/METADATA +1 -1
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/RECORD +8 -8
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/WHEEL +0 -0
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/entry_points.txt +0 -0
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/licenses/LICENSE +0 -0
- {mobile_mcp_ai-2.6.7.dist-info → mobile_mcp_ai-2.6.10.dist-info}/top_level.txt +0 -0
|
@@ -14,10 +14,108 @@ import asyncio
|
|
|
14
14
|
import time
|
|
15
15
|
import re
|
|
16
16
|
from pathlib import Path
|
|
17
|
-
from typing import Dict, List, Optional
|
|
17
|
+
from typing import Dict, List, Optional, Tuple
|
|
18
18
|
from datetime import datetime
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
# ==================== 弹窗检测配置常量 ====================
|
|
22
|
+
|
|
23
|
+
# 确认性按钮文本(最高优先级 - 优先点击这些按钮关闭弹窗)
|
|
24
|
+
CONFIRM_BUTTON_TEXTS = [
|
|
25
|
+
# 中文确认类
|
|
26
|
+
'同意', '同意并继续', '同意协议', '我同意', '同意条款',
|
|
27
|
+
'确认', '确定', '好的', '好', '知道了', '我知道了', '明白了',
|
|
28
|
+
'允许', '始终允许', '仅在使用中允许', '仅本次允许',
|
|
29
|
+
'立即体验', '立即开始', '开始使用', '马上体验',
|
|
30
|
+
'去设置', '立即更新', '立即升级', '现在更新',
|
|
31
|
+
'OK', 'Ok', 'ok', 'Yes', 'YES', 'yes',
|
|
32
|
+
'Accept', 'ACCEPT', 'accept', 'Agree', 'AGREE', 'agree',
|
|
33
|
+
'Allow', 'ALLOW', 'allow', 'Confirm', 'CONFIRM', 'confirm',
|
|
34
|
+
'Got it', 'GOT IT', 'Done', 'DONE', 'Continue', 'CONTINUE',
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
# 关闭/取消按钮文本(次优先级 - 当没有确认按钮时使用)
|
|
38
|
+
CLOSE_BUTTON_TEXTS = [
|
|
39
|
+
# 符号类
|
|
40
|
+
'×', 'X', 'x', '✕', '✖', '╳',
|
|
41
|
+
# 中文关闭/拒绝类
|
|
42
|
+
'关闭', '取消', '跳过', '放弃', '算了', '忽略', '不了', '拒绝', '否',
|
|
43
|
+
'稍后再说', '下次再说', '以后再说', '暂不需要', '不再提示',
|
|
44
|
+
'不要', '残忍拒绝', '狠心离开', '我再想想', '暂不',
|
|
45
|
+
'不感兴趣', '跳过广告', '稍后更新',
|
|
46
|
+
'不同意', '仍不同意', '不同意并退出', # 这些优先级最低
|
|
47
|
+
# 英文关闭/拒绝类
|
|
48
|
+
'close', 'Close', 'CLOSE', 'Skip', 'skip', 'SKIP',
|
|
49
|
+
'Dismiss', 'dismiss', 'Cancel', 'cancel', 'CANCEL',
|
|
50
|
+
'Not now', 'NOT NOW', 'Later', 'later', 'LATER',
|
|
51
|
+
'No thanks', 'NO THANKS', 'No', 'NO', 'no',
|
|
52
|
+
'Ignore', 'IGNORE', 'Deny', 'DENY', 'Reject', 'REJECT',
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
# 关闭按钮描述关键词(模糊匹配 content-desc / resource-id)
|
|
56
|
+
# 确认类关键词(优先)
|
|
57
|
+
CONFIRM_BUTTON_KEYWORDS = [
|
|
58
|
+
'confirm', 'accept', 'agree', 'allow', 'ok', 'yes', 'done',
|
|
59
|
+
'确认', '同意', '允许', '确定',
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
# 关闭类关键词(次优先)
|
|
63
|
+
CLOSE_BUTTON_KEYWORDS = [
|
|
64
|
+
'close', 'dismiss', 'cancel', 'skip', 'ignore', 'deny', 'reject',
|
|
65
|
+
'关闭', '取消', '跳过', '忽略', '拒绝',
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
# 弹窗检测 - class 名称关键词(强特征)
|
|
69
|
+
POPUP_CLASS_KEYWORDS = [
|
|
70
|
+
'Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet',
|
|
71
|
+
'PopupWindow', 'DialogFragment', 'AlertDialog',
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
# 弹窗检测 - resource-id 关键词(强特征)
|
|
75
|
+
POPUP_ID_KEYWORDS = [
|
|
76
|
+
'dialog', 'popup', 'alert', 'modal', 'bottom_sheet',
|
|
77
|
+
'overlay', 'mask', 'dim', 'scrim',
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
# 广告弹窗 resource-id 关键词
|
|
81
|
+
AD_POPUP_KEYWORDS = [
|
|
82
|
+
'ad_close', 'ad_button', 'full_screen', 'interstitial',
|
|
83
|
+
'reward', 'close_icon', 'close_btn', 'ad_container',
|
|
84
|
+
'splash', 'banner', 'native_ad',
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
# 小元素排除关键词(避免误点)
|
|
88
|
+
EXCLUDE_ELEMENT_KEYWORDS = [
|
|
89
|
+
# 功能按钮
|
|
90
|
+
'more', 'menu', 'setting', 'settings', 'option', 'options',
|
|
91
|
+
'share', 'favorite', 'like', 'comment', 'follow', 'download',
|
|
92
|
+
'search', 'back', 'home', 'profile', 'notification', 'message',
|
|
93
|
+
# 播放控制
|
|
94
|
+
'play', 'pause', 'next', 'previous', 'volume', 'fullscreen',
|
|
95
|
+
'mute', 'unmute', 'forward', 'rewind', 'seek',
|
|
96
|
+
# 导航
|
|
97
|
+
'tab', 'navigation', 'drawer', 'sidebar',
|
|
98
|
+
# 编辑
|
|
99
|
+
'edit', 'delete', 'copy', 'paste', 'cut', 'select',
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
# 页面内容特征关键词(非弹窗)
|
|
103
|
+
PAGE_CONTENT_KEYWORDS = [
|
|
104
|
+
'video', 'player', 'recycler', 'list', 'scroll',
|
|
105
|
+
'viewpager', 'fragment', 'webview', 'content',
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
# 弹窗检测默认置信度阈值
|
|
109
|
+
DEFAULT_POPUP_CONFIDENCE_THRESHOLD = 0.6
|
|
110
|
+
|
|
111
|
+
# iOS 弹窗元素类型
|
|
112
|
+
IOS_POPUP_TYPES = [
|
|
113
|
+
'XCUIElementTypeAlert',
|
|
114
|
+
'XCUIElementTypeSheet',
|
|
115
|
+
'XCUIElementTypePopover',
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
|
|
21
119
|
class BasicMobileToolsLite:
|
|
22
120
|
"""精简版移动端工具"""
|
|
23
121
|
|
|
@@ -511,11 +609,15 @@ class BasicMobileToolsLite:
|
|
|
511
609
|
"""截图并添加网格坐标标注(用于精确定位元素)
|
|
512
610
|
|
|
513
611
|
在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
|
|
514
|
-
|
|
612
|
+
|
|
613
|
+
🎯 弹窗检测场景:
|
|
614
|
+
- show_popup_hints=True: 明确弹窗场景时使用(如调用 mobile_close_popup 前)
|
|
615
|
+
- show_popup_hints=False: 普通截图,不检测弹窗
|
|
515
616
|
|
|
516
617
|
Args:
|
|
517
618
|
grid_size: 网格间距(像素),默认 100。建议值:50-200
|
|
518
|
-
show_popup_hints: 是否显示弹窗关闭按钮提示位置,默认
|
|
619
|
+
show_popup_hints: 是否显示弹窗关闭按钮提示位置,默认 False
|
|
620
|
+
仅在明确弹窗场景时设置为 True,会标注弹窗区域和可能的关闭按钮位置。
|
|
519
621
|
|
|
520
622
|
Returns:
|
|
521
623
|
包含标注截图路径和弹窗信息的字典
|
|
@@ -681,12 +783,17 @@ class BasicMobileToolsLite:
|
|
|
681
783
|
except Exception as e:
|
|
682
784
|
return {"success": False, "message": f"❌ 网格截图失败: {e}"}
|
|
683
785
|
|
|
684
|
-
def take_screenshot_with_som(self) -> Dict:
|
|
786
|
+
def take_screenshot_with_som(self, check_popup: bool = False) -> Dict:
|
|
685
787
|
"""Set-of-Mark 截图:给每个可点击元素标上数字(超级好用!)
|
|
686
788
|
|
|
687
789
|
在截图上给每个可点击元素画框并标上数字编号。
|
|
688
790
|
AI 看图后直接说"点击 3 号",然后调用 click_by_som(3) 即可。
|
|
689
791
|
|
|
792
|
+
Args:
|
|
793
|
+
check_popup: 是否检测弹窗,默认 False
|
|
794
|
+
- True: 明确弹窗场景时使用(如调用 mobile_close_popup 前)
|
|
795
|
+
- False: 普通截图,不检测弹窗
|
|
796
|
+
|
|
690
797
|
Returns:
|
|
691
798
|
包含标注截图和元素列表的字典
|
|
692
799
|
"""
|
|
@@ -823,11 +930,12 @@ class BasicMobileToolsLite:
|
|
|
823
930
|
'resource_id': elem.get('resource_id', '')
|
|
824
931
|
})
|
|
825
932
|
|
|
826
|
-
# 第3.5
|
|
933
|
+
# 第3.5步:检测弹窗区域(仅在明确弹窗场景时检测)
|
|
827
934
|
popup_bounds = None
|
|
828
935
|
popup_confidence = 0
|
|
829
936
|
|
|
830
|
-
|
|
937
|
+
# 🎯 明确弹窗检测场景:只在 check_popup=True 时检测(明确弹窗场景)
|
|
938
|
+
if check_popup and not self._is_ios():
|
|
831
939
|
try:
|
|
832
940
|
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
833
941
|
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
@@ -884,7 +992,53 @@ class BasicMobileToolsLite:
|
|
|
884
992
|
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
885
993
|
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
886
994
|
|
|
887
|
-
|
|
995
|
+
# 第5步:AI 分析图片内容(使用 Cursor AI 视觉能力)
|
|
996
|
+
ai_analysis = None
|
|
997
|
+
try:
|
|
998
|
+
ai_analysis = self._analyze_screenshot_with_ai(
|
|
999
|
+
str(final_path), screen_width, screen_height, img_width, img_height
|
|
1000
|
+
)
|
|
1001
|
+
except Exception as e:
|
|
1002
|
+
# AI 分析失败不影响主功能
|
|
1003
|
+
pass
|
|
1004
|
+
|
|
1005
|
+
# 构建返回消息
|
|
1006
|
+
message = f"📸 SoM 截图已保存: {final_path}\n"
|
|
1007
|
+
message += f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
1008
|
+
message += f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
1009
|
+
|
|
1010
|
+
# 添加 AI 分析提示(实际分析由 Cursor AI 在看到图片后自动完成)
|
|
1011
|
+
if ai_analysis:
|
|
1012
|
+
screenshot_path_for_ai = ai_analysis.get('screenshot_path', str(final_path))
|
|
1013
|
+
message += f"\n🤖 【AI 视觉分析请求】\n"
|
|
1014
|
+
message += f"📸 截图路径: {screenshot_path_for_ai}\n"
|
|
1015
|
+
message += f"📐 屏幕尺寸: {screen_width}x{screen_height},图片尺寸: {img_width}x{img_height}\n\n"
|
|
1016
|
+
message += f"请分析这张截图:\n"
|
|
1017
|
+
message += f"1. 识别页面主要内容\n"
|
|
1018
|
+
message += f"2. 检测是否有弹窗(对话框、提示框、广告、教程弹窗等)\n"
|
|
1019
|
+
message += f"3. 如果有弹窗,识别关闭按钮位置(×、关闭、知道了、好的、同意等)\n"
|
|
1020
|
+
message += f"4. 返回关闭按钮的精确位置:\n"
|
|
1021
|
+
message += f" - 像素坐标 (x, y)\n"
|
|
1022
|
+
message += f" - 百分比坐标 (x_percent, y_percent)\n"
|
|
1023
|
+
message += f" - 按钮文本(如'知道了'、'好的'等)\n\n"
|
|
1024
|
+
message += f"💡 分析结果格式(JSON):\n"
|
|
1025
|
+
message += f"{{\n"
|
|
1026
|
+
message += f" \"has_popup\": true/false,\n"
|
|
1027
|
+
message += f" \"popup_description\": \"弹窗描述\",\n"
|
|
1028
|
+
message += f" \"close_button\": {{\n"
|
|
1029
|
+
message += f" \"x\": 像素X坐标,\n"
|
|
1030
|
+
message += f" \"y\": 像素Y坐标,\n"
|
|
1031
|
+
message += f" \"x_percent\": X百分比(0-100),\n"
|
|
1032
|
+
message += f" \"y_percent\": Y百分比(0-100),\n"
|
|
1033
|
+
message += f" \"text\": \"按钮文本\"\n"
|
|
1034
|
+
message += f" }}\n"
|
|
1035
|
+
message += f"}}\n"
|
|
1036
|
+
|
|
1037
|
+
message += f"💡 使用方法:\n"
|
|
1038
|
+
message += f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
1039
|
+
message += f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
1040
|
+
|
|
1041
|
+
result = {
|
|
888
1042
|
"success": True,
|
|
889
1043
|
"screenshot_path": str(final_path),
|
|
890
1044
|
"screen_width": screen_width,
|
|
@@ -895,19 +1049,82 @@ class BasicMobileToolsLite:
|
|
|
895
1049
|
"elements": som_elements,
|
|
896
1050
|
"popup_detected": popup_bounds is not None,
|
|
897
1051
|
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
|
|
898
|
-
"message":
|
|
899
|
-
f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
900
|
-
f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
901
|
-
f"💡 使用方法:\n"
|
|
902
|
-
f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
903
|
-
f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
1052
|
+
"message": message
|
|
904
1053
|
}
|
|
905
1054
|
|
|
1055
|
+
# 添加 AI 分析结果到返回字典
|
|
1056
|
+
if ai_analysis:
|
|
1057
|
+
result["ai_analysis"] = ai_analysis
|
|
1058
|
+
|
|
1059
|
+
return result
|
|
1060
|
+
|
|
906
1061
|
except ImportError:
|
|
907
1062
|
return {"success": False, "message": "❌ 需要安装 Pillow: pip install Pillow"}
|
|
908
1063
|
except Exception as e:
|
|
909
1064
|
return {"success": False, "message": f"❌ SoM 截图失败: {e}"}
|
|
910
1065
|
|
|
1066
|
+
def _analyze_screenshot_with_ai(self, screenshot_path: str, screen_width: int, screen_height: int,
|
|
1067
|
+
image_width: int, image_height: int) -> Optional[Dict]:
|
|
1068
|
+
"""准备 AI 分析截图内容
|
|
1069
|
+
|
|
1070
|
+
在 MCP 环境中,返回图片路径和分析提示,让 Cursor AI 自动分析图片。
|
|
1071
|
+
实际的 AI 分析由 Cursor AI 在看到图片后自动完成。
|
|
1072
|
+
|
|
1073
|
+
Args:
|
|
1074
|
+
screenshot_path: 截图文件路径
|
|
1075
|
+
screen_width: 屏幕宽度
|
|
1076
|
+
screen_height: 屏幕高度
|
|
1077
|
+
image_width: 图片宽度
|
|
1078
|
+
image_height: 图片高度
|
|
1079
|
+
|
|
1080
|
+
Returns:
|
|
1081
|
+
AI 分析准备信息字典
|
|
1082
|
+
"""
|
|
1083
|
+
try:
|
|
1084
|
+
from pathlib import Path
|
|
1085
|
+
|
|
1086
|
+
# 检查图片文件是否存在
|
|
1087
|
+
img_path = Path(screenshot_path)
|
|
1088
|
+
if not img_path.exists():
|
|
1089
|
+
return None
|
|
1090
|
+
|
|
1091
|
+
# 返回分析准备信息
|
|
1092
|
+
# 注意:实际的 AI 分析由 Cursor AI 在看到图片后自动完成
|
|
1093
|
+
return {
|
|
1094
|
+
"needs_ai_analysis": True,
|
|
1095
|
+
"screenshot_path": screenshot_path,
|
|
1096
|
+
"screen_size": {"width": screen_width, "height": screen_height},
|
|
1097
|
+
"image_size": {"width": image_width, "height": image_height},
|
|
1098
|
+
"analysis_instructions": (
|
|
1099
|
+
"🤖 【AI 视觉分析请求】\n"
|
|
1100
|
+
"请分析截图文件: {}\n\n"
|
|
1101
|
+
"分析要求:\n"
|
|
1102
|
+
"1. 识别页面主要内容\n"
|
|
1103
|
+
"2. 检测是否有弹窗(对话框、提示框、广告、教程弹窗等)\n"
|
|
1104
|
+
"3. 如果有弹窗,识别关闭按钮位置(×、关闭、知道了、好的、同意等)\n"
|
|
1105
|
+
"4. 返回关闭按钮的精确位置:\n"
|
|
1106
|
+
" - 像素坐标 (x, y)\n"
|
|
1107
|
+
" - 百分比坐标 (x_percent, y_percent)\n"
|
|
1108
|
+
" - 按钮文本(如'知道了'、'好的'等)\n\n"
|
|
1109
|
+
"屏幕尺寸: {}x{},图片尺寸: {}x{}\n\n"
|
|
1110
|
+
"请以 JSON 格式返回分析结果:\n"
|
|
1111
|
+
"{{\n"
|
|
1112
|
+
" \"has_popup\": true/false,\n"
|
|
1113
|
+
" \"popup_description\": \"弹窗描述\",\n"
|
|
1114
|
+
" \"close_button\": {{\n"
|
|
1115
|
+
" \"x\": 像素X坐标,\n"
|
|
1116
|
+
" \"y\": 像素Y坐标,\n"
|
|
1117
|
+
" \"x_percent\": X百分比(0-100),\n"
|
|
1118
|
+
" \"y_percent\": Y百分比(0-100),\n"
|
|
1119
|
+
" \"text\": \"按钮文本\"\n"
|
|
1120
|
+
" }}\n"
|
|
1121
|
+
"}}"
|
|
1122
|
+
).format(screenshot_path, screen_width, screen_height, image_width, image_height)
|
|
1123
|
+
}
|
|
1124
|
+
except Exception as e:
|
|
1125
|
+
# AI 分析准备失败不影响主功能
|
|
1126
|
+
return None
|
|
1127
|
+
|
|
911
1128
|
def click_by_som(self, index: int) -> Dict:
|
|
912
1129
|
"""根据 SoM 编号点击元素
|
|
913
1130
|
|
|
@@ -991,7 +1208,23 @@ class BasicMobileToolsLite:
|
|
|
991
1208
|
}
|
|
992
1209
|
|
|
993
1210
|
except Exception as e:
|
|
994
|
-
|
|
1211
|
+
# 🎯 异常情况:点击失败时检测弹窗
|
|
1212
|
+
popup_detected = False
|
|
1213
|
+
popup_hint = ""
|
|
1214
|
+
if not self._is_ios():
|
|
1215
|
+
try:
|
|
1216
|
+
page_analysis = self._analyze_page_for_popup()
|
|
1217
|
+
if page_analysis.get("has_popup"):
|
|
1218
|
+
popup_detected = True
|
|
1219
|
+
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1220
|
+
except Exception:
|
|
1221
|
+
pass
|
|
1222
|
+
|
|
1223
|
+
error_msg = f"❌ 点击失败: {e}\n💡 如果页面已变化,请重新调用 mobile_screenshot_with_som 刷新元素列表"
|
|
1224
|
+
if popup_detected:
|
|
1225
|
+
error_msg += popup_hint
|
|
1226
|
+
|
|
1227
|
+
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
995
1228
|
|
|
996
1229
|
def _take_screenshot_no_compress(self, description: str = "") -> Dict:
|
|
997
1230
|
"""截图(不压缩,PIL 不可用时的备用方案)"""
|
|
@@ -1177,7 +1410,23 @@ class BasicMobileToolsLite:
|
|
|
1177
1410
|
"return_to_app": return_result
|
|
1178
1411
|
}
|
|
1179
1412
|
except Exception as e:
|
|
1180
|
-
|
|
1413
|
+
# 🎯 异常情况:点击失败时检测弹窗
|
|
1414
|
+
popup_detected = False
|
|
1415
|
+
popup_hint = ""
|
|
1416
|
+
if not self._is_ios():
|
|
1417
|
+
try:
|
|
1418
|
+
page_analysis = self._analyze_page_for_popup()
|
|
1419
|
+
if page_analysis.get("has_popup"):
|
|
1420
|
+
popup_detected = True
|
|
1421
|
+
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1422
|
+
except Exception:
|
|
1423
|
+
pass
|
|
1424
|
+
|
|
1425
|
+
error_msg = f"❌ 点击失败: {e}"
|
|
1426
|
+
if popup_detected:
|
|
1427
|
+
error_msg += popup_hint
|
|
1428
|
+
|
|
1429
|
+
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
1181
1430
|
|
|
1182
1431
|
def click_by_percent(self, x_percent: float, y_percent: float) -> Dict:
|
|
1183
1432
|
"""通过百分比坐标点击(跨设备兼容)
|
|
@@ -1335,7 +1584,23 @@ class BasicMobileToolsLite:
|
|
|
1335
1584
|
|
|
1336
1585
|
return {"success": False, "message": f"❌ 文本不存在: {text}"}
|
|
1337
1586
|
except Exception as e:
|
|
1338
|
-
|
|
1587
|
+
# 🎯 异常情况:点击失败时检测弹窗
|
|
1588
|
+
popup_detected = False
|
|
1589
|
+
popup_hint = ""
|
|
1590
|
+
if not self._is_ios():
|
|
1591
|
+
try:
|
|
1592
|
+
page_analysis = self._analyze_page_for_popup()
|
|
1593
|
+
if page_analysis.get("has_popup"):
|
|
1594
|
+
popup_detected = True
|
|
1595
|
+
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1596
|
+
except Exception:
|
|
1597
|
+
pass
|
|
1598
|
+
|
|
1599
|
+
error_msg = f"❌ 点击失败: {e}"
|
|
1600
|
+
if popup_detected:
|
|
1601
|
+
error_msg += popup_hint
|
|
1602
|
+
|
|
1603
|
+
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
1339
1604
|
|
|
1340
1605
|
def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
|
|
1341
1606
|
"""在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
|
|
@@ -1516,7 +1781,23 @@ class BasicMobileToolsLite:
|
|
|
1516
1781
|
return {"success": False, "message": f"❌ 索引超出范围: 找到 {count} 个元素,但请求索引 {index}"}
|
|
1517
1782
|
return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
|
|
1518
1783
|
except Exception as e:
|
|
1519
|
-
|
|
1784
|
+
# 🎯 异常情况:点击失败时检测弹窗
|
|
1785
|
+
popup_detected = False
|
|
1786
|
+
popup_hint = ""
|
|
1787
|
+
if not self._is_ios():
|
|
1788
|
+
try:
|
|
1789
|
+
page_analysis = self._analyze_page_for_popup()
|
|
1790
|
+
if page_analysis.get("has_popup"):
|
|
1791
|
+
popup_detected = True
|
|
1792
|
+
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1793
|
+
except Exception:
|
|
1794
|
+
pass
|
|
1795
|
+
|
|
1796
|
+
error_msg = f"❌ 点击失败: {e}"
|
|
1797
|
+
if popup_detected:
|
|
1798
|
+
error_msg += popup_hint
|
|
1799
|
+
|
|
1800
|
+
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
1520
1801
|
|
|
1521
1802
|
# ==================== 长按操作 ====================
|
|
1522
1803
|
|
|
@@ -2271,10 +2552,177 @@ class BasicMobileToolsLite:
|
|
|
2271
2552
|
time.sleep(seconds)
|
|
2272
2553
|
return {"success": True, "message": f"✅ 已等待 {seconds} 秒"}
|
|
2273
2554
|
|
|
2555
|
+
async def drag_progress_bar(self, direction: str = "right", distance_percent: float = 30.0,
|
|
2556
|
+
y_percent: Optional[float] = None, y: Optional[int] = None) -> Dict:
|
|
2557
|
+
"""智能拖动进度条
|
|
2558
|
+
|
|
2559
|
+
自动检测进度条是否可见:
|
|
2560
|
+
- 如果进度条已显示,直接拖动(无需先点击播放区域)
|
|
2561
|
+
- 如果进度条未显示,先点击播放区域显示控制栏,再拖动
|
|
2562
|
+
|
|
2563
|
+
Args:
|
|
2564
|
+
direction: 拖动方向,'left'(倒退)或 'right'(前进),默认 'right'
|
|
2565
|
+
distance_percent: 拖动距离百分比 (0-100),默认 30%
|
|
2566
|
+
y_percent: 进度条的垂直位置百分比 (0-100),如果未指定则自动检测
|
|
2567
|
+
y: 进度条的垂直位置坐标(像素),如果未指定则自动检测
|
|
2568
|
+
"""
|
|
2569
|
+
try:
|
|
2570
|
+
import xml.etree.ElementTree as ET
|
|
2571
|
+
import re
|
|
2572
|
+
|
|
2573
|
+
if self._is_ios():
|
|
2574
|
+
return {"success": False, "message": "❌ iOS 暂不支持,请使用 mobile_swipe"}
|
|
2575
|
+
|
|
2576
|
+
if direction not in ['left', 'right']:
|
|
2577
|
+
return {"success": False, "message": f"❌ 拖动方向必须是 'left' 或 'right': {direction}"}
|
|
2578
|
+
|
|
2579
|
+
screen_width, screen_height = self.client.u2.window_size()
|
|
2580
|
+
|
|
2581
|
+
# 获取 XML 查找进度条
|
|
2582
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2583
|
+
root = ET.fromstring(xml_string)
|
|
2584
|
+
|
|
2585
|
+
progress_bar_found = False
|
|
2586
|
+
progress_bar_y = None
|
|
2587
|
+
progress_bar_y_percent = None
|
|
2588
|
+
|
|
2589
|
+
# 查找进度条元素(SeekBar、ProgressBar)
|
|
2590
|
+
for elem in root.iter():
|
|
2591
|
+
class_name = elem.attrib.get('class', '')
|
|
2592
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
2593
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2594
|
+
|
|
2595
|
+
# 检查是否是进度条
|
|
2596
|
+
is_progress_bar = (
|
|
2597
|
+
'SeekBar' in class_name or
|
|
2598
|
+
'ProgressBar' in class_name or
|
|
2599
|
+
'progress' in resource_id.lower() or
|
|
2600
|
+
'seek' in resource_id.lower()
|
|
2601
|
+
)
|
|
2602
|
+
|
|
2603
|
+
if is_progress_bar and bounds_str:
|
|
2604
|
+
# 解析 bounds 获取进度条位置
|
|
2605
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2606
|
+
if match:
|
|
2607
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2608
|
+
center_y = (y1 + y2) // 2
|
|
2609
|
+
progress_bar_y = center_y
|
|
2610
|
+
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2611
|
+
progress_bar_found = True
|
|
2612
|
+
break
|
|
2613
|
+
|
|
2614
|
+
# 如果未找到进度条,尝试点击播放区域显示控制栏
|
|
2615
|
+
if not progress_bar_found:
|
|
2616
|
+
# 点击屏幕中心显示控制栏
|
|
2617
|
+
center_x, center_y = screen_width // 2, screen_height // 2
|
|
2618
|
+
self.client.u2.click(center_x, center_y)
|
|
2619
|
+
time.sleep(0.5)
|
|
2620
|
+
|
|
2621
|
+
# 再次查找进度条
|
|
2622
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2623
|
+
root = ET.fromstring(xml_string)
|
|
2624
|
+
|
|
2625
|
+
for elem in root.iter():
|
|
2626
|
+
class_name = elem.attrib.get('class', '')
|
|
2627
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
2628
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2629
|
+
|
|
2630
|
+
is_progress_bar = (
|
|
2631
|
+
'SeekBar' in class_name or
|
|
2632
|
+
'ProgressBar' in class_name or
|
|
2633
|
+
'progress' in resource_id.lower() or
|
|
2634
|
+
'seek' in resource_id.lower()
|
|
2635
|
+
)
|
|
2636
|
+
|
|
2637
|
+
if is_progress_bar and bounds_str:
|
|
2638
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2639
|
+
if match:
|
|
2640
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2641
|
+
center_y = (y1 + y2) // 2
|
|
2642
|
+
progress_bar_y = center_y
|
|
2643
|
+
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2644
|
+
progress_bar_found = True
|
|
2645
|
+
break
|
|
2646
|
+
|
|
2647
|
+
# 确定使用的高度位置
|
|
2648
|
+
if y_percent is not None:
|
|
2649
|
+
swipe_y = int(screen_height * y_percent / 100)
|
|
2650
|
+
used_y_percent = y_percent
|
|
2651
|
+
elif y is not None:
|
|
2652
|
+
swipe_y = y
|
|
2653
|
+
used_y_percent = round(y / screen_height * 100, 1)
|
|
2654
|
+
elif progress_bar_found:
|
|
2655
|
+
swipe_y = progress_bar_y
|
|
2656
|
+
used_y_percent = progress_bar_y_percent
|
|
2657
|
+
else:
|
|
2658
|
+
# 默认使用屏幕底部附近(进度条常见位置)
|
|
2659
|
+
swipe_y = int(screen_height * 0.91)
|
|
2660
|
+
used_y_percent = 91.0
|
|
2661
|
+
|
|
2662
|
+
# 计算滑动距离
|
|
2663
|
+
swipe_distance = int(screen_width * distance_percent / 100)
|
|
2664
|
+
|
|
2665
|
+
# 计算起始和结束位置
|
|
2666
|
+
center_x = screen_width // 2
|
|
2667
|
+
if direction == 'left':
|
|
2668
|
+
start_x = min(center_x + swipe_distance // 2, screen_width - 10)
|
|
2669
|
+
end_x = start_x - swipe_distance
|
|
2670
|
+
if end_x < 10:
|
|
2671
|
+
end_x = 10
|
|
2672
|
+
start_x = min(end_x + swipe_distance, screen_width - 10)
|
|
2673
|
+
else: # right
|
|
2674
|
+
start_x = max(center_x - swipe_distance // 2, 10)
|
|
2675
|
+
end_x = start_x + swipe_distance
|
|
2676
|
+
if end_x > screen_width - 10:
|
|
2677
|
+
end_x = screen_width - 10
|
|
2678
|
+
start_x = max(end_x - swipe_distance, 10)
|
|
2679
|
+
|
|
2680
|
+
# 执行拖动
|
|
2681
|
+
self.client.u2.swipe(start_x, swipe_y, end_x, swipe_y, duration=0.5)
|
|
2682
|
+
time.sleep(0.3)
|
|
2683
|
+
|
|
2684
|
+
# 记录操作
|
|
2685
|
+
self._record_swipe(direction)
|
|
2686
|
+
|
|
2687
|
+
# 检查应用是否跳转
|
|
2688
|
+
app_check = self._check_app_switched()
|
|
2689
|
+
return_result = None
|
|
2690
|
+
if app_check['switched']:
|
|
2691
|
+
return_result = self._return_to_target_app()
|
|
2692
|
+
|
|
2693
|
+
# 构建返回消息
|
|
2694
|
+
msg = f"✅ 进度条拖动成功: {direction} (高度: {used_y_percent}%, 距离: {distance_percent}%)"
|
|
2695
|
+
if not progress_bar_found:
|
|
2696
|
+
msg += "\n💡 已自动点击播放区域显示控制栏"
|
|
2697
|
+
else:
|
|
2698
|
+
msg += "\n💡 进度条已显示,直接拖动"
|
|
2699
|
+
|
|
2700
|
+
if app_check['switched']:
|
|
2701
|
+
msg += f"\n{app_check['message']}"
|
|
2702
|
+
if return_result and return_result.get('success'):
|
|
2703
|
+
msg += f"\n{return_result['message']}"
|
|
2704
|
+
|
|
2705
|
+
return {
|
|
2706
|
+
"success": True,
|
|
2707
|
+
"message": msg,
|
|
2708
|
+
"progress_bar_found": progress_bar_found,
|
|
2709
|
+
"y_percent": used_y_percent,
|
|
2710
|
+
"distance_percent": distance_percent,
|
|
2711
|
+
"direction": direction,
|
|
2712
|
+
"app_check": app_check,
|
|
2713
|
+
"return_to_app": return_result
|
|
2714
|
+
}
|
|
2715
|
+
|
|
2716
|
+
except Exception as e:
|
|
2717
|
+
return {"success": False, "message": f"❌ 拖动进度条失败: {e}"}
|
|
2718
|
+
|
|
2274
2719
|
# ==================== 应用管理 ====================
|
|
2275
2720
|
|
|
2276
2721
|
async def launch_app(self, package_name: str) -> Dict:
|
|
2277
|
-
"""启动应用
|
|
2722
|
+
"""启动应用
|
|
2723
|
+
|
|
2724
|
+
启动应用后会自动检测弹窗(启动应用场景)
|
|
2725
|
+
"""
|
|
2278
2726
|
try:
|
|
2279
2727
|
if self._is_ios():
|
|
2280
2728
|
ios_client = self._get_ios_client()
|
|
@@ -2298,12 +2746,45 @@ class BasicMobileToolsLite:
|
|
|
2298
2746
|
|
|
2299
2747
|
self._record_operation('launch_app', package_name=package_name)
|
|
2300
2748
|
|
|
2749
|
+
# 🎯 启动应用后检测弹窗(明确场景:启动应用后)
|
|
2750
|
+
popup_detected = False
|
|
2751
|
+
popup_message = ""
|
|
2752
|
+
if not self._is_ios():
|
|
2753
|
+
try:
|
|
2754
|
+
# 检测弹窗
|
|
2755
|
+
page_analysis = self._analyze_page_for_popup()
|
|
2756
|
+
if page_analysis.get("has_popup"):
|
|
2757
|
+
popup_detected = True
|
|
2758
|
+
popup_info = page_analysis.get("popup_info", {})
|
|
2759
|
+
popup_message = f"\n🎯 检测到弹窗(启动应用场景)\n💡 如需关闭弹窗,请调用 mobile_close_popup()"
|
|
2760
|
+
except Exception:
|
|
2761
|
+
pass # 弹窗检测失败不影响启动流程
|
|
2762
|
+
|
|
2763
|
+
message = f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载\n📱 已设置应用状态监测"
|
|
2764
|
+
if popup_detected:
|
|
2765
|
+
message += popup_message
|
|
2766
|
+
|
|
2301
2767
|
return {
|
|
2302
2768
|
"success": True,
|
|
2303
|
-
"message":
|
|
2769
|
+
"message": message,
|
|
2770
|
+
"popup_detected": popup_detected
|
|
2304
2771
|
}
|
|
2305
2772
|
except Exception as e:
|
|
2306
|
-
|
|
2773
|
+
# 🎯 异常情况:启动失败时也检测弹窗
|
|
2774
|
+
popup_detected = False
|
|
2775
|
+
if not self._is_ios():
|
|
2776
|
+
try:
|
|
2777
|
+
page_analysis = self._analyze_page_for_popup()
|
|
2778
|
+
if page_analysis.get("has_popup"):
|
|
2779
|
+
popup_detected = True
|
|
2780
|
+
except Exception:
|
|
2781
|
+
pass
|
|
2782
|
+
|
|
2783
|
+
error_msg = f"❌ 启动失败: {e}"
|
|
2784
|
+
if popup_detected:
|
|
2785
|
+
error_msg += "\n🎯 检测到弹窗(异常情况),可能影响启动,建议先关闭弹窗"
|
|
2786
|
+
|
|
2787
|
+
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
2307
2788
|
|
|
2308
2789
|
def terminate_app(self, package_name: str) -> Dict:
|
|
2309
2790
|
"""终止应用"""
|
|
@@ -2602,31 +3083,38 @@ class BasicMobileToolsLite:
|
|
|
2602
3083
|
|
|
2603
3084
|
# 策略3:小尺寸的 clickable 元素(可能是 X 图标)
|
|
2604
3085
|
elif clickable:
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
3086
|
+
# 排除非关闭按钮的关键词(避免误识别更多按钮、菜单按钮等)
|
|
3087
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
3088
|
+
exclude_keywords = ['more', 'menu', 'setting', 'settings', 'option', 'options',
|
|
3089
|
+
'share', 'favorite', 'like', 'comment', 'follow', 'download']
|
|
3090
|
+
if any(kw in resource_id.lower() for kw in exclude_keywords):
|
|
3091
|
+
score = 0 # 排除这些元素
|
|
3092
|
+
else:
|
|
3093
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
3094
|
+
max_size = max(120, int(screen_width * 0.12))
|
|
3095
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
3096
|
+
# 基于位置评分:角落位置加分
|
|
3097
|
+
rel_x = center_x / screen_width
|
|
3098
|
+
rel_y = center_y / screen_height
|
|
3099
|
+
|
|
3100
|
+
# 右上角得分最高
|
|
3101
|
+
if rel_x > 0.6 and rel_y < 0.5:
|
|
3102
|
+
score = 70 + (rel_x - 0.6) * 50 + (0.5 - rel_y) * 50
|
|
3103
|
+
reason = f"右上角小元素 {width}x{height}px"
|
|
3104
|
+
# 左上角
|
|
3105
|
+
elif rel_x < 0.4 and rel_y < 0.5:
|
|
3106
|
+
score = 60 + (0.4 - rel_x) * 50 + (0.5 - rel_y) * 50
|
|
3107
|
+
reason = f"左上角小元素 {width}x{height}px"
|
|
3108
|
+
# 其他位置的小元素
|
|
3109
|
+
elif 'Image' in class_name:
|
|
3110
|
+
score = 50
|
|
3111
|
+
reason = f"图片元素 {width}x{height}px"
|
|
3112
|
+
else:
|
|
3113
|
+
score = 40
|
|
3114
|
+
reason = f"小型可点击元素 {width}x{height}px"
|
|
2627
3115
|
|
|
2628
3116
|
if score > 0:
|
|
2629
|
-
|
|
3117
|
+
candidate = {
|
|
2630
3118
|
'score': score,
|
|
2631
3119
|
'reason': reason,
|
|
2632
3120
|
'bounds': bounds_str,
|
|
@@ -2634,44 +3122,58 @@ class BasicMobileToolsLite:
|
|
|
2634
3122
|
'center_y': center_y,
|
|
2635
3123
|
'x_percent': x_percent,
|
|
2636
3124
|
'y_percent': y_percent,
|
|
2637
|
-
'size': f"{width}x{height}"
|
|
2638
|
-
|
|
3125
|
+
'size': f"{width}x{height}",
|
|
3126
|
+
'text': text,
|
|
3127
|
+
'content_desc': content_desc,
|
|
3128
|
+
'class': class_name,
|
|
3129
|
+
'clickable': clickable
|
|
3130
|
+
}
|
|
3131
|
+
# 尝试获取 resource-id(如果存在)
|
|
3132
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
3133
|
+
if resource_id:
|
|
3134
|
+
candidate['resource_id'] = resource_id
|
|
3135
|
+
candidates.append(candidate)
|
|
2639
3136
|
|
|
2640
3137
|
if not candidates:
|
|
2641
|
-
#
|
|
2642
|
-
|
|
3138
|
+
# 没找到,不截图(优化:避免频繁截图)
|
|
3139
|
+
# 调用者可以使用 mobile_close_popup() 作为降级方案
|
|
2643
3140
|
return {
|
|
2644
3141
|
"success": False,
|
|
2645
|
-
"message": "❌
|
|
2646
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3142
|
+
"message": "❌ 元素树未找到关闭按钮",
|
|
2647
3143
|
"screen_size": {"width": screen_width, "height": screen_height},
|
|
2648
|
-
"
|
|
2649
|
-
"width": screenshot_result.get("image_width"),
|
|
2650
|
-
"height": screenshot_result.get("image_height")
|
|
2651
|
-
},
|
|
2652
|
-
"original_size": {
|
|
2653
|
-
"width": screenshot_result.get("original_img_width"),
|
|
2654
|
-
"height": screenshot_result.get("original_img_height")
|
|
2655
|
-
},
|
|
2656
|
-
"tip": "请分析截图找到 X 关闭按钮,然后调用 mobile_click_by_percent(x_percent, y_percent)"
|
|
3144
|
+
"tip": "建议:1) 使用 mobile_close_popup() 作为降级方案(可能截图) 2) 或手动调用 mobile_screenshot_with_som() 进行AI分析"
|
|
2657
3145
|
}
|
|
2658
3146
|
|
|
2659
3147
|
# 按得分排序
|
|
2660
3148
|
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2661
3149
|
best = candidates[0]
|
|
2662
3150
|
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
"message": f"✅ 找到可能的关闭按钮",
|
|
2666
|
-
"best_candidate": {
|
|
3151
|
+
# 构建关闭按钮信息(优化:包含更多点击方式)
|
|
3152
|
+
close_button_info = {
|
|
2667
3153
|
"reason": best['reason'],
|
|
2668
3154
|
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
2669
3155
|
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2670
3156
|
"bounds": best['bounds'],
|
|
2671
3157
|
"size": best['size'],
|
|
2672
3158
|
"score": best['score']
|
|
2673
|
-
|
|
2674
|
-
|
|
3159
|
+
}
|
|
3160
|
+
|
|
3161
|
+
# 添加可用的点击方式(优化:优先使用 resource_id 或 text)
|
|
3162
|
+
if best.get('resource_id'):
|
|
3163
|
+
close_button_info['resource_id'] = best['resource_id']
|
|
3164
|
+
click_command = f"mobile_click_by_id('{best['resource_id']}')"
|
|
3165
|
+
elif best.get('text'):
|
|
3166
|
+
close_button_info['text'] = best['text']
|
|
3167
|
+
click_command = f"mobile_click_by_text('{best['text']}')"
|
|
3168
|
+
else:
|
|
3169
|
+
click_command = f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})"
|
|
3170
|
+
|
|
3171
|
+
return {
|
|
3172
|
+
"success": True,
|
|
3173
|
+
"message": f"✅ 找到可能的关闭按钮",
|
|
3174
|
+
"close_button": close_button_info, # 优化:统一字段名
|
|
3175
|
+
"best_candidate": close_button_info, # 保持向后兼容
|
|
3176
|
+
"click_command": click_command,
|
|
2675
3177
|
"other_candidates": [
|
|
2676
3178
|
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
2677
3179
|
for c in candidates[1:4]
|
|
@@ -2682,363 +3184,733 @@ class BasicMobileToolsLite:
|
|
|
2682
3184
|
except Exception as e:
|
|
2683
3185
|
return {"success": False, "message": f"❌ 查找关闭按钮失败: {e}"}
|
|
2684
3186
|
|
|
2685
|
-
def
|
|
2686
|
-
"""
|
|
2687
|
-
|
|
2688
|
-
核心改进:先检测弹窗区域,再在弹窗范围内查找关闭按钮
|
|
2689
|
-
|
|
2690
|
-
策略(优先级从高到低):
|
|
2691
|
-
1. 检测弹窗区域(非全屏的大面积容器)
|
|
2692
|
-
2. 在弹窗边界内查找关闭相关的文本/描述(×、X、关闭、close 等)
|
|
2693
|
-
3. 在弹窗边界内查找小尺寸的 clickable 元素(优先边角位置)
|
|
2694
|
-
4. 如果都找不到,截图让 AI 视觉识别
|
|
3187
|
+
def _analyze_page_for_popup(self) -> Dict:
|
|
3188
|
+
"""分析页面是否有弹窗(用于启动应用后、异常情况等场景)
|
|
2695
3189
|
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
- 使用百分比坐标记录,跨分辨率兼容
|
|
3190
|
+
Returns:
|
|
3191
|
+
包含 has_popup 和 popup_info 的字典
|
|
2699
3192
|
"""
|
|
2700
3193
|
try:
|
|
2701
|
-
import re
|
|
2702
|
-
import xml.etree.ElementTree as ET
|
|
2703
|
-
|
|
2704
|
-
# 获取屏幕尺寸
|
|
2705
3194
|
if self._is_ios():
|
|
2706
|
-
return {"
|
|
2707
|
-
|
|
2708
|
-
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2709
|
-
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3195
|
+
return {"has_popup": False, "popup_info": None}
|
|
2710
3196
|
|
|
2711
|
-
#
|
|
2712
|
-
|
|
3197
|
+
# 获取元素列表
|
|
3198
|
+
elements = self.list_elements()
|
|
2713
3199
|
|
|
2714
|
-
#
|
|
2715
|
-
|
|
2716
|
-
close_desc_keywords = ['关闭', 'close', 'dismiss', 'cancel', '跳过']
|
|
3200
|
+
# 分析页面内容
|
|
3201
|
+
page_analysis = self._detect_page_content(elements)
|
|
2717
3202
|
|
|
2718
|
-
|
|
2719
|
-
|
|
3203
|
+
return {
|
|
3204
|
+
"has_popup": page_analysis.get("has_popup", False),
|
|
3205
|
+
"popup_info": page_analysis.get("popup_info")
|
|
3206
|
+
}
|
|
3207
|
+
except Exception:
|
|
3208
|
+
return {"has_popup": False, "popup_info": None}
|
|
3209
|
+
|
|
3210
|
+
def _detect_page_content(self, elements: List[Dict],
|
|
3211
|
+
confidence_threshold: float = None,
|
|
3212
|
+
xml_string: str = None) -> Dict:
|
|
3213
|
+
"""基于元素列表识别页面内容
|
|
3214
|
+
|
|
3215
|
+
使用全局常量 CONFIRM_BUTTON_TEXTS、CLOSE_BUTTON_TEXTS、
|
|
3216
|
+
CONFIRM_BUTTON_KEYWORDS、CLOSE_BUTTON_KEYWORDS
|
|
3217
|
+
|
|
3218
|
+
Args:
|
|
3219
|
+
elements: 元素列表
|
|
3220
|
+
confidence_threshold: 弹窗检测置信度阈值,默认使用 DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3221
|
+
xml_string: 预先获取的 XML 字符串(可选,避免重复调用)
|
|
2720
3222
|
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
3223
|
+
Returns:
|
|
3224
|
+
{
|
|
3225
|
+
"main_content": {...}, # 主要功能区域
|
|
3226
|
+
"has_popup": bool, # 是否有弹窗
|
|
3227
|
+
"popup_info": {...}, # 弹窗信息
|
|
3228
|
+
"interactive_elements": [...] # 可交互元素
|
|
3229
|
+
}
|
|
3230
|
+
"""
|
|
3231
|
+
if confidence_threshold is None:
|
|
3232
|
+
confidence_threshold = DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3233
|
+
|
|
3234
|
+
if self._is_ios():
|
|
3235
|
+
return self._detect_page_content_ios(elements, confidence_threshold)
|
|
3236
|
+
|
|
3237
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3238
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3239
|
+
|
|
3240
|
+
# 获取可交互元素
|
|
3241
|
+
interactive_elements = [
|
|
3242
|
+
elem for elem in elements
|
|
3243
|
+
if elem.get("clickable") is True or str(elem.get("clickable", "")).lower() == "true"
|
|
3244
|
+
]
|
|
3245
|
+
|
|
3246
|
+
# 分析是否有弹窗(基于元素特征)
|
|
3247
|
+
has_popup = False
|
|
3248
|
+
popup_info = None
|
|
3249
|
+
|
|
3250
|
+
# 通过XML检测弹窗(更准确)
|
|
3251
|
+
try:
|
|
3252
|
+
if xml_string is None:
|
|
3253
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
3254
|
+
import xml.etree.ElementTree as ET
|
|
3255
|
+
root = ET.fromstring(xml_string)
|
|
3256
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3257
|
+
root, screen_width, screen_height
|
|
3258
|
+
)
|
|
3259
|
+
if popup_bounds and popup_confidence >= confidence_threshold:
|
|
3260
|
+
has_popup = True
|
|
3261
|
+
popup_info = {
|
|
3262
|
+
"bounds": popup_bounds,
|
|
3263
|
+
"confidence": popup_confidence,
|
|
3264
|
+
"detected_by": "xml_structure"
|
|
3265
|
+
}
|
|
3266
|
+
except Exception:
|
|
3267
|
+
pass
|
|
3268
|
+
|
|
3269
|
+
# 如果没有通过XML检测到,检查是否有明显的弹窗按钮特征
|
|
3270
|
+
if not has_popup:
|
|
3271
|
+
for elem in interactive_elements:
|
|
3272
|
+
text = elem.get("text", "")
|
|
3273
|
+
resource_id = elem.get("resource_id", "")
|
|
3274
|
+
content_desc = elem.get("content_desc", "")
|
|
2725
3275
|
|
|
2726
|
-
#
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
3276
|
+
# 检查文本是否匹配确认按钮(优先检测)
|
|
3277
|
+
if text in CONFIRM_BUTTON_TEXTS:
|
|
3278
|
+
has_popup = True
|
|
3279
|
+
popup_info = {"detected_by": "confirm_button_text", "text": text}
|
|
3280
|
+
break
|
|
2730
3281
|
|
|
2731
|
-
#
|
|
2732
|
-
|
|
3282
|
+
# 检查文本是否匹配关闭按钮
|
|
3283
|
+
if text in CLOSE_BUTTON_TEXTS:
|
|
3284
|
+
has_popup = True
|
|
3285
|
+
popup_info = {"detected_by": "close_button_text", "text": text}
|
|
3286
|
+
break
|
|
2733
3287
|
|
|
2734
|
-
#
|
|
2735
|
-
|
|
3288
|
+
# 检查 resource-id 是否包含确认关键词
|
|
3289
|
+
if any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3290
|
+
has_popup = True
|
|
3291
|
+
popup_info = {"detected_by": "confirm_button_id", "resource_id": resource_id}
|
|
3292
|
+
break
|
|
2736
3293
|
|
|
2737
|
-
#
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
class_name = elem.attrib.get('class', '')
|
|
2743
|
-
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2744
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2745
|
-
|
|
2746
|
-
if not bounds_str:
|
|
2747
|
-
continue
|
|
2748
|
-
|
|
2749
|
-
# 解析 bounds
|
|
2750
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2751
|
-
if not match:
|
|
2752
|
-
continue
|
|
2753
|
-
|
|
2754
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2755
|
-
width = x2 - x1
|
|
2756
|
-
height = y2 - y1
|
|
2757
|
-
center_x = (x1 + x2) // 2
|
|
2758
|
-
center_y = (y1 + y2) // 2
|
|
2759
|
-
|
|
2760
|
-
# 如果检测到弹窗区域,检查元素是否在弹窗范围内或附近
|
|
2761
|
-
in_popup = False
|
|
2762
|
-
popup_edge_bonus = 0
|
|
2763
|
-
is_floating_close = False # 是否是浮动关闭按钮(在弹窗外部上方)
|
|
2764
|
-
if popup_bounds:
|
|
2765
|
-
px1, py1, px2, py2 = popup_bounds
|
|
2766
|
-
|
|
2767
|
-
# 关闭按钮可能在弹窗外部(常见设计:X 按钮浮在弹窗右上角外侧)
|
|
2768
|
-
# 扩大搜索范围:弹窗上方 200 像素,右侧 50 像素
|
|
2769
|
-
margin_top = 200 # 上方扩展范围(关闭按钮常在弹窗上方)
|
|
2770
|
-
margin_side = 50 # 左右扩展范围
|
|
2771
|
-
margin_bottom = 30 # 下方扩展范围
|
|
2772
|
-
|
|
2773
|
-
in_popup = (px1 - margin_side <= center_x <= px2 + margin_side and
|
|
2774
|
-
py1 - margin_top <= center_y <= py2 + margin_bottom)
|
|
2775
|
-
|
|
2776
|
-
# 检查是否是浮动关闭按钮(在弹窗外侧:上方或下方)
|
|
2777
|
-
# 上方浮动关闭按钮(常见:右上角外侧)
|
|
2778
|
-
if center_y < py1 and center_y > py1 - margin_top:
|
|
2779
|
-
if center_x > (px1 + px2) / 2: # 在弹窗右半部分上方
|
|
2780
|
-
is_floating_close = True
|
|
2781
|
-
# 下方浮动关闭按钮(常见:底部中间外侧)
|
|
2782
|
-
elif center_y > py2 and center_y < py2 + margin_top:
|
|
2783
|
-
# 下方关闭按钮通常在中间位置
|
|
2784
|
-
if abs(center_x - (px1 + px2) / 2) < (px2 - px1) / 2:
|
|
2785
|
-
is_floating_close = True
|
|
2786
|
-
|
|
2787
|
-
if in_popup:
|
|
2788
|
-
# 计算元素是否在弹窗边缘(关闭按钮通常在边缘)
|
|
2789
|
-
dist_to_top = abs(center_y - py1)
|
|
2790
|
-
dist_to_bottom = abs(center_y - py2)
|
|
2791
|
-
dist_to_left = abs(center_x - px1)
|
|
2792
|
-
dist_to_right = abs(center_x - px2)
|
|
2793
|
-
min_dist = min(dist_to_top, dist_to_bottom, dist_to_left, dist_to_right)
|
|
2794
|
-
|
|
2795
|
-
# 在弹窗边缘 100 像素内的元素加分
|
|
2796
|
-
if min_dist < 100:
|
|
2797
|
-
popup_edge_bonus = 3.0 * (1 - min_dist / 100)
|
|
2798
|
-
|
|
2799
|
-
# 浮动关闭按钮(在弹窗上方外侧)给予高额加分
|
|
2800
|
-
if is_floating_close:
|
|
2801
|
-
popup_edge_bonus += 5.0 # 大幅加分
|
|
2802
|
-
elif not popup_detected:
|
|
2803
|
-
# 没有检测到弹窗时,只处理有明确关闭特征的元素
|
|
2804
|
-
# 检查是否有明确的关闭特征(文本、resource-id、content-desc)
|
|
2805
|
-
has_explicit_close_feature = (
|
|
2806
|
-
text in close_texts or
|
|
2807
|
-
any(kw in content_desc.lower() for kw in close_desc_keywords) or
|
|
2808
|
-
'close' in resource_id.lower() or
|
|
2809
|
-
'dismiss' in resource_id.lower() or
|
|
2810
|
-
'cancel' in resource_id.lower()
|
|
2811
|
-
)
|
|
2812
|
-
if not has_explicit_close_feature:
|
|
2813
|
-
continue # 没有明确关闭特征,跳过
|
|
2814
|
-
# 有明确关闭特征时,允许处理
|
|
2815
|
-
in_popup = True
|
|
2816
|
-
|
|
2817
|
-
if not in_popup:
|
|
2818
|
-
continue
|
|
2819
|
-
|
|
2820
|
-
# 相对位置(0-1)
|
|
2821
|
-
rel_x = center_x / screen_width
|
|
2822
|
-
rel_y = center_y / screen_height
|
|
2823
|
-
|
|
2824
|
-
score = 0
|
|
2825
|
-
match_type = ""
|
|
2826
|
-
position = self._get_position_name(rel_x, rel_y)
|
|
2827
|
-
|
|
2828
|
-
# ===== 策略1:精确匹配关闭文本(最高优先级)=====
|
|
2829
|
-
if text in close_texts:
|
|
2830
|
-
score = 15.0 + popup_edge_bonus
|
|
2831
|
-
match_type = f"text='{text}'"
|
|
2832
|
-
|
|
2833
|
-
# ===== 策略2:content-desc 包含关闭关键词 =====
|
|
2834
|
-
elif any(kw in content_desc.lower() for kw in close_desc_keywords):
|
|
2835
|
-
score = 12.0 + popup_edge_bonus
|
|
2836
|
-
match_type = f"desc='{content_desc}'"
|
|
2837
|
-
|
|
2838
|
-
# ===== 策略3:clickable 的小尺寸元素(优先于非 clickable)=====
|
|
2839
|
-
elif clickable:
|
|
2840
|
-
min_size = max(20, int(screen_width * 0.03))
|
|
2841
|
-
max_size = max(120, int(screen_width * 0.15))
|
|
2842
|
-
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2843
|
-
# clickable 元素基础分更高
|
|
2844
|
-
base_score = 8.0
|
|
2845
|
-
# 浮动关闭按钮给予最高分
|
|
2846
|
-
if is_floating_close:
|
|
2847
|
-
base_score = 12.0
|
|
2848
|
-
match_type = "floating_close"
|
|
2849
|
-
elif 'Image' in class_name:
|
|
2850
|
-
score = base_score + 2.0
|
|
2851
|
-
match_type = "clickable_image"
|
|
2852
|
-
else:
|
|
2853
|
-
match_type = "clickable"
|
|
2854
|
-
score = base_score + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2855
|
-
|
|
2856
|
-
# ===== 策略4:ImageView/ImageButton 类型的小元素(非 clickable)=====
|
|
2857
|
-
elif 'Image' in class_name:
|
|
2858
|
-
min_size = max(15, int(screen_width * 0.02))
|
|
2859
|
-
max_size = max(120, int(screen_width * 0.12))
|
|
2860
|
-
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2861
|
-
score = 5.0 + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2862
|
-
match_type = "ImageView"
|
|
2863
|
-
|
|
2864
|
-
# XML 顺序加分(后出现的元素在上层,更可能是弹窗内的元素)
|
|
2865
|
-
if score > 0:
|
|
2866
|
-
xml_order_bonus = idx / len(all_elements) * 2.0 # 最多加 2 分
|
|
2867
|
-
score += xml_order_bonus
|
|
2868
|
-
|
|
2869
|
-
close_candidates.append({
|
|
2870
|
-
'bounds': bounds_str,
|
|
2871
|
-
'center_x': center_x,
|
|
2872
|
-
'center_y': center_y,
|
|
2873
|
-
'width': width,
|
|
2874
|
-
'height': height,
|
|
2875
|
-
'score': score,
|
|
2876
|
-
'position': position,
|
|
2877
|
-
'match_type': match_type,
|
|
2878
|
-
'text': text,
|
|
2879
|
-
'content_desc': content_desc,
|
|
2880
|
-
'x_percent': round(rel_x * 100, 1),
|
|
2881
|
-
'y_percent': round(rel_y * 100, 1),
|
|
2882
|
-
'in_popup': popup_detected
|
|
2883
|
-
})
|
|
2884
|
-
|
|
2885
|
-
except ET.ParseError:
|
|
2886
|
-
pass
|
|
2887
|
-
|
|
2888
|
-
if not close_candidates:
|
|
2889
|
-
# 如果检测到高置信度的弹窗区域,先尝试点击常见的关闭按钮位置
|
|
2890
|
-
if popup_detected and popup_bounds:
|
|
2891
|
-
px1, py1, px2, py2 = popup_bounds
|
|
2892
|
-
popup_width = px2 - px1
|
|
2893
|
-
popup_height = py2 - py1
|
|
2894
|
-
|
|
2895
|
-
# 【优化】X按钮有三种常见位置:
|
|
2896
|
-
# 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
|
|
2897
|
-
# 2. 弹窗边界上方(浮动X按钮)
|
|
2898
|
-
# 3. 弹窗正下方(底部关闭按钮)
|
|
2899
|
-
offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
|
|
2900
|
-
offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
|
|
2901
|
-
offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
|
|
2902
|
-
|
|
2903
|
-
try_positions = [
|
|
2904
|
-
# 【最高优先级】弹窗内紧贴顶部边界
|
|
2905
|
-
(px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
|
|
2906
|
-
# 弹窗边界上方(浮动X按钮)
|
|
2907
|
-
(px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
|
|
2908
|
-
# 弹窗正下方中间(底部关闭按钮)
|
|
2909
|
-
((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
|
|
2910
|
-
# 弹窗正上方中间
|
|
2911
|
-
((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
|
|
2912
|
-
]
|
|
2913
|
-
|
|
2914
|
-
for try_x, try_y, position_name in try_positions:
|
|
2915
|
-
if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
|
|
2916
|
-
self.client.u2.click(try_x, try_y)
|
|
2917
|
-
time.sleep(0.3)
|
|
2918
|
-
|
|
2919
|
-
# 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
|
|
2920
|
-
app_check = self._check_app_switched()
|
|
2921
|
-
return_result = None
|
|
2922
|
-
|
|
2923
|
-
if app_check['switched']:
|
|
2924
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
2925
|
-
return_result = self._return_to_target_app()
|
|
2926
|
-
|
|
2927
|
-
# 尝试后截图,让 AI 判断是否成功
|
|
2928
|
-
screenshot_result = self.take_screenshot("尝试关闭后")
|
|
2929
|
-
|
|
2930
|
-
msg = f"✅ 已尝试点击常见关闭按钮位置"
|
|
2931
|
-
if app_check['switched']:
|
|
2932
|
-
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
2933
|
-
if return_result:
|
|
2934
|
-
if return_result['success']:
|
|
2935
|
-
msg += f"\n{return_result['message']}"
|
|
2936
|
-
else:
|
|
2937
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2938
|
-
|
|
2939
|
-
return {
|
|
2940
|
-
"success": True,
|
|
2941
|
-
"message": msg,
|
|
2942
|
-
"tried_positions": [p[2] for p in try_positions],
|
|
2943
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2944
|
-
"app_check": app_check,
|
|
2945
|
-
"return_to_app": return_result,
|
|
2946
|
-
"tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
|
|
2947
|
-
}
|
|
3294
|
+
# 检查 resource-id 是否包含关闭关键词
|
|
3295
|
+
if any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3296
|
+
has_popup = True
|
|
3297
|
+
popup_info = {"detected_by": "close_button_id", "resource_id": resource_id}
|
|
3298
|
+
break
|
|
2948
3299
|
|
|
2949
|
-
#
|
|
2950
|
-
|
|
3300
|
+
# 检查 content-desc 是否包含确认关键词
|
|
3301
|
+
if any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3302
|
+
has_popup = True
|
|
3303
|
+
popup_info = {"detected_by": "confirm_button_desc", "content_desc": content_desc}
|
|
3304
|
+
break
|
|
2951
3305
|
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
"
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
3306
|
+
# 检查 content-desc 是否包含关闭关键词
|
|
3307
|
+
if any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3308
|
+
has_popup = True
|
|
3309
|
+
popup_info = {"detected_by": "close_button_desc", "content_desc": content_desc}
|
|
3310
|
+
break
|
|
3311
|
+
|
|
3312
|
+
return {
|
|
3313
|
+
"has_popup": has_popup,
|
|
3314
|
+
"popup_info": popup_info,
|
|
3315
|
+
"interactive_elements": interactive_elements,
|
|
3316
|
+
"total_elements": len(elements)
|
|
3317
|
+
}
|
|
3318
|
+
|
|
3319
|
+
def _detect_page_content_ios(self, elements: List[Dict], confidence_threshold: float) -> Dict:
|
|
3320
|
+
"""iOS 平台基于元素列表识别页面内容
|
|
3321
|
+
|
|
3322
|
+
Args:
|
|
3323
|
+
elements: 元素列表
|
|
3324
|
+
confidence_threshold: 弹窗检测置信度阈值
|
|
3325
|
+
|
|
3326
|
+
Returns:
|
|
3327
|
+
页面内容分析结果
|
|
3328
|
+
"""
|
|
3329
|
+
# 获取可交互元素
|
|
3330
|
+
interactive_elements = [
|
|
3331
|
+
elem for elem in elements
|
|
3332
|
+
if elem.get("type") in ['XCUIElementTypeButton', 'XCUIElementTypeLink', 'XCUIElementTypeCell']
|
|
3333
|
+
]
|
|
3334
|
+
|
|
3335
|
+
has_popup = False
|
|
3336
|
+
popup_info = None
|
|
3337
|
+
|
|
3338
|
+
# iOS 弹窗检测:检查是否有 Alert/Sheet 类型元素
|
|
3339
|
+
for elem in elements:
|
|
3340
|
+
elem_type = elem.get("type", "")
|
|
3341
|
+
if elem_type in IOS_POPUP_TYPES:
|
|
3342
|
+
has_popup = True
|
|
3343
|
+
popup_info = {
|
|
3344
|
+
"detected_by": "ios_popup_type",
|
|
3345
|
+
"type": elem_type,
|
|
3346
|
+
"bounds": elem.get("bounds")
|
|
2968
3347
|
}
|
|
3348
|
+
break
|
|
3349
|
+
|
|
3350
|
+
# 如果没有检测到系统弹窗,检查是否有确认/关闭按钮文本
|
|
3351
|
+
if not has_popup:
|
|
3352
|
+
for elem in interactive_elements:
|
|
3353
|
+
text = elem.get("label", "") or elem.get("text", "") or ""
|
|
3354
|
+
# 优先检测确认按钮
|
|
3355
|
+
if text in CONFIRM_BUTTON_TEXTS:
|
|
3356
|
+
has_popup = True
|
|
3357
|
+
popup_info = {"detected_by": "confirm_button_text", "text": text}
|
|
3358
|
+
break
|
|
3359
|
+
# 其次检测关闭按钮
|
|
3360
|
+
if text in CLOSE_BUTTON_TEXTS:
|
|
3361
|
+
has_popup = True
|
|
3362
|
+
popup_info = {"detected_by": "close_button_text", "text": text}
|
|
3363
|
+
break
|
|
3364
|
+
|
|
3365
|
+
return {
|
|
3366
|
+
"has_popup": has_popup,
|
|
3367
|
+
"popup_info": popup_info,
|
|
3368
|
+
"interactive_elements": interactive_elements,
|
|
3369
|
+
"total_elements": len(elements)
|
|
3370
|
+
}
|
|
3371
|
+
|
|
3372
|
+
def _find_close_button_in_elements(self, elements: List[Dict], page_analysis: Dict = None) -> Optional[Dict]:
|
|
3373
|
+
"""在元素列表中查找关闭按钮
|
|
3374
|
+
|
|
3375
|
+
优先级顺序:
|
|
3376
|
+
1. 确认性按钮(同意、确认、允许等)- 最高优先级
|
|
3377
|
+
2. 关闭/取消按钮(关闭、取消、跳过等)- 次优先级
|
|
3378
|
+
3. 小尺寸可点击元素(可能是 X 图标)- 最低优先级
|
|
3379
|
+
|
|
3380
|
+
使用全局常量 CONFIRM_BUTTON_TEXTS、CLOSE_BUTTON_TEXTS、
|
|
3381
|
+
CONFIRM_BUTTON_KEYWORDS、CLOSE_BUTTON_KEYWORDS、EXCLUDE_ELEMENT_KEYWORDS
|
|
3382
|
+
|
|
3383
|
+
Args:
|
|
3384
|
+
elements: 元素列表
|
|
3385
|
+
page_analysis: 页面分析结果(可选)
|
|
2969
3386
|
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
3387
|
+
Returns:
|
|
3388
|
+
关闭按钮信息,如果未找到返回None
|
|
3389
|
+
"""
|
|
3390
|
+
if self._is_ios():
|
|
3391
|
+
return self._find_close_button_in_elements_ios(elements)
|
|
3392
|
+
|
|
3393
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3394
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3395
|
+
|
|
3396
|
+
candidates = []
|
|
3397
|
+
|
|
3398
|
+
for elem in elements:
|
|
3399
|
+
text = elem.get("text", "").strip()
|
|
3400
|
+
content_desc = elem.get("content_desc", "").strip()
|
|
3401
|
+
resource_id = elem.get("resource_id", "").strip()
|
|
3402
|
+
bounds = elem.get("bounds", "")
|
|
3403
|
+
clickable = elem.get("clickable", False)
|
|
3404
|
+
|
|
3405
|
+
if not bounds:
|
|
3406
|
+
continue
|
|
2973
3407
|
|
|
2974
|
-
#
|
|
2975
|
-
|
|
2976
|
-
|
|
3408
|
+
# 解析 bounds
|
|
3409
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
3410
|
+
if not match:
|
|
3411
|
+
continue
|
|
2977
3412
|
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
3413
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
3414
|
+
width = x2 - x1
|
|
3415
|
+
height = y2 - y1
|
|
3416
|
+
center_x = (x1 + x2) // 2
|
|
3417
|
+
center_y = (y1 + y2) // 2
|
|
3418
|
+
|
|
3419
|
+
# 计算百分比
|
|
3420
|
+
x_percent = round(center_x / screen_width * 100, 1)
|
|
3421
|
+
y_percent = round(center_y / screen_height * 100, 1)
|
|
3422
|
+
|
|
3423
|
+
score = 0
|
|
3424
|
+
reason = ""
|
|
3425
|
+
|
|
3426
|
+
# 策略1:确认性按钮文本(最高优先级)- 优先点击同意、确认等
|
|
3427
|
+
if text in CONFIRM_BUTTON_TEXTS:
|
|
3428
|
+
score = 120 # 最高分
|
|
3429
|
+
reason = f"确认按钮文本='{text}'"
|
|
3430
|
+
|
|
3431
|
+
# 策略2:确认类 resource-id 匹配
|
|
3432
|
+
elif any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3433
|
+
score = 115
|
|
3434
|
+
reason = f"确认按钮ID='{resource_id}'"
|
|
3435
|
+
|
|
3436
|
+
# 策略3:确认类 content-desc 匹配
|
|
3437
|
+
elif any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3438
|
+
score = 110
|
|
3439
|
+
reason = f"确认按钮描述='{content_desc}'"
|
|
3440
|
+
|
|
3441
|
+
# 策略4:关闭/取消按钮文本(次优先级)
|
|
3442
|
+
elif text in CLOSE_BUTTON_TEXTS:
|
|
3443
|
+
score = 100
|
|
3444
|
+
reason = f"关闭按钮文本='{text}'"
|
|
3445
|
+
|
|
3446
|
+
# 策略5:关闭类 resource-id 匹配
|
|
3447
|
+
elif any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3448
|
+
score = 95
|
|
3449
|
+
reason = f"关闭按钮ID='{resource_id}'"
|
|
3450
|
+
|
|
3451
|
+
# 策略6:关闭类 content-desc 匹配
|
|
3452
|
+
elif any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3453
|
+
score = 90
|
|
3454
|
+
reason = f"关闭按钮描述='{content_desc}'"
|
|
3455
|
+
|
|
3456
|
+
# 策略7:小尺寸的 clickable 元素(可能是 X 图标)- 最低优先级
|
|
3457
|
+
elif clickable:
|
|
3458
|
+
# 排除非关闭按钮的关键词 - 使用全局常量
|
|
3459
|
+
if any(kw in resource_id.lower() for kw in EXCLUDE_ELEMENT_KEYWORDS):
|
|
3460
|
+
score = 0 # 排除这些元素
|
|
3461
|
+
elif any(kw in content_desc.lower() for kw in EXCLUDE_ELEMENT_KEYWORDS):
|
|
3462
|
+
score = 0 # 排除这些元素
|
|
3463
|
+
else:
|
|
3464
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
3465
|
+
max_size = max(120, int(screen_width * 0.15))
|
|
3466
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
3467
|
+
rel_x = center_x / screen_width
|
|
3468
|
+
rel_y = center_y / screen_height
|
|
3469
|
+
|
|
3470
|
+
# 右上角得分最高
|
|
3471
|
+
if rel_x > 0.6 and rel_y < 0.5:
|
|
3472
|
+
score = 70 + (rel_x - 0.6) * 50 + (0.5 - rel_y) * 50
|
|
3473
|
+
reason = f"右上角小元素 {width}x{height}px"
|
|
3474
|
+
# 左上角
|
|
3475
|
+
elif rel_x < 0.4 and rel_y < 0.5:
|
|
3476
|
+
score = 60 + (0.4 - rel_x) * 50 + (0.5 - rel_y) * 50
|
|
3477
|
+
reason = f"左上角小元素 {width}x{height}px"
|
|
3478
|
+
else:
|
|
3479
|
+
score = 40
|
|
3480
|
+
reason = f"小型可点击元素 {width}x{height}px"
|
|
2981
3481
|
|
|
2982
|
-
if
|
|
2983
|
-
|
|
2984
|
-
|
|
3482
|
+
if score > 0:
|
|
3483
|
+
candidates.append({
|
|
3484
|
+
'score': score,
|
|
3485
|
+
'reason': reason,
|
|
3486
|
+
'bounds': bounds,
|
|
3487
|
+
'center_x': center_x,
|
|
3488
|
+
'center_y': center_y,
|
|
3489
|
+
'x_percent': x_percent,
|
|
3490
|
+
'y_percent': y_percent,
|
|
3491
|
+
'text': text,
|
|
3492
|
+
'resource_id': resource_id,
|
|
3493
|
+
'content_desc': content_desc
|
|
3494
|
+
})
|
|
3495
|
+
|
|
3496
|
+
if not candidates:
|
|
3497
|
+
return None
|
|
3498
|
+
|
|
3499
|
+
# 按得分排序,返回最佳候选(确认按钮优先)
|
|
3500
|
+
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3501
|
+
return candidates[0]
|
|
3502
|
+
|
|
3503
|
+
def _find_close_button_in_elements_ios(self, elements: List[Dict]) -> Optional[Dict]:
|
|
3504
|
+
"""iOS 平台在元素列表中查找关闭按钮
|
|
3505
|
+
|
|
3506
|
+
优先级顺序:
|
|
3507
|
+
1. 确认性按钮(同意、确认、允许等)- 最高优先级
|
|
3508
|
+
2. 关闭/取消按钮(关闭、取消、跳过等)- 次优先级
|
|
3509
|
+
3. 小尺寸可点击元素(可能是 X 图标)- 最低优先级
|
|
3510
|
+
|
|
3511
|
+
Args:
|
|
3512
|
+
elements: 元素列表
|
|
3513
|
+
|
|
3514
|
+
Returns:
|
|
3515
|
+
关闭按钮信息,如果未找到返回None
|
|
3516
|
+
"""
|
|
3517
|
+
ios_client = self._get_ios_client()
|
|
3518
|
+
if not ios_client:
|
|
3519
|
+
return None
|
|
3520
|
+
|
|
3521
|
+
try:
|
|
3522
|
+
screen_size = ios_client.window_size()
|
|
3523
|
+
screen_width = screen_size.width
|
|
3524
|
+
screen_height = screen_size.height
|
|
3525
|
+
except:
|
|
3526
|
+
screen_width = 375
|
|
3527
|
+
screen_height = 812
|
|
3528
|
+
|
|
3529
|
+
candidates = []
|
|
3530
|
+
|
|
3531
|
+
for elem in elements:
|
|
3532
|
+
text = elem.get("text", "") or elem.get("label", "") or ""
|
|
3533
|
+
text = text.strip()
|
|
3534
|
+
elem_type = elem.get("type", "")
|
|
3535
|
+
bounds = elem.get("bounds", {})
|
|
3536
|
+
|
|
3537
|
+
# iOS bounds 格式不同
|
|
3538
|
+
if isinstance(bounds, dict):
|
|
3539
|
+
x = bounds.get("x", 0)
|
|
3540
|
+
y = bounds.get("y", 0)
|
|
3541
|
+
width = bounds.get("width", 0)
|
|
3542
|
+
height = bounds.get("height", 0)
|
|
3543
|
+
else:
|
|
3544
|
+
continue
|
|
3545
|
+
|
|
3546
|
+
center_x = int(x + width / 2)
|
|
3547
|
+
center_y = int(y + height / 2)
|
|
3548
|
+
x_percent = round(center_x / screen_width * 100, 1)
|
|
3549
|
+
y_percent = round(center_y / screen_height * 100, 1)
|
|
3550
|
+
|
|
3551
|
+
score = 0
|
|
3552
|
+
reason = ""
|
|
3553
|
+
|
|
3554
|
+
# 策略1:确认性按钮文本(最高优先级)
|
|
3555
|
+
if text in CONFIRM_BUTTON_TEXTS:
|
|
3556
|
+
score = 120
|
|
3557
|
+
reason = f"确认按钮文本='{text}'"
|
|
3558
|
+
|
|
3559
|
+
# 策略2:确认类关键词匹配
|
|
3560
|
+
elif any(kw in text.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3561
|
+
score = 115
|
|
3562
|
+
reason = f"确认按钮关键词: '{text}'"
|
|
3563
|
+
|
|
3564
|
+
# 策略3:关闭/取消按钮文本(次优先级)
|
|
3565
|
+
elif text in CLOSE_BUTTON_TEXTS:
|
|
3566
|
+
score = 100
|
|
3567
|
+
reason = f"关闭按钮文本='{text}'"
|
|
3568
|
+
|
|
3569
|
+
# 策略4:关闭类关键词匹配
|
|
3570
|
+
elif any(kw in text.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3571
|
+
score = 95
|
|
3572
|
+
reason = f"关闭按钮关键词: '{text}'"
|
|
3573
|
+
|
|
3574
|
+
# 策略5:XCUIElementTypeButton 类型的小元素(最低优先级)
|
|
3575
|
+
elif elem_type == 'XCUIElementTypeButton':
|
|
3576
|
+
if 20 <= width <= 60 and 20 <= height <= 60:
|
|
3577
|
+
rel_x = center_x / screen_width
|
|
3578
|
+
rel_y = center_y / screen_height
|
|
3579
|
+
if rel_x > 0.7 and rel_y < 0.3:
|
|
3580
|
+
score = 70
|
|
3581
|
+
reason = f"右上角小按钮 {width}x{height}px"
|
|
3582
|
+
|
|
3583
|
+
if score > 0:
|
|
3584
|
+
candidates.append({
|
|
3585
|
+
'score': score,
|
|
3586
|
+
'reason': reason,
|
|
3587
|
+
'bounds': f"[{x},{y}][{x+width},{y+height}]",
|
|
3588
|
+
'center_x': center_x,
|
|
3589
|
+
'center_y': center_y,
|
|
3590
|
+
'x_percent': x_percent,
|
|
3591
|
+
'y_percent': y_percent,
|
|
3592
|
+
'text': text,
|
|
3593
|
+
'resource_id': '',
|
|
3594
|
+
'content_desc': ''
|
|
3595
|
+
})
|
|
3596
|
+
|
|
3597
|
+
if not candidates:
|
|
3598
|
+
return None
|
|
3599
|
+
|
|
3600
|
+
# 按得分排序(确认按钮优先)
|
|
3601
|
+
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3602
|
+
return candidates[0]
|
|
3603
|
+
|
|
3604
|
+
def _close_popup_by_elements(self, elements: List[Dict] = None,
|
|
3605
|
+
xml_string: str = None,
|
|
3606
|
+
confidence_threshold: float = None,
|
|
3607
|
+
page_fingerprint_before: Dict = None) -> Dict:
|
|
3608
|
+
"""阶段1:通过控件树关闭弹窗(最快、最可靠)
|
|
3609
|
+
|
|
3610
|
+
Args:
|
|
3611
|
+
elements: 预获取的元素列表(可选,避免重复调用)
|
|
3612
|
+
xml_string: 预获取的 XML 字符串(可选,避免重复调用)
|
|
3613
|
+
confidence_threshold: 弹窗检测置信度阈值
|
|
3614
|
+
page_fingerprint_before: 操作前的页面指纹(用于验证)
|
|
2985
3615
|
|
|
2986
|
-
|
|
2987
|
-
|
|
3616
|
+
Returns:
|
|
3617
|
+
关闭结果
|
|
3618
|
+
"""
|
|
3619
|
+
try:
|
|
3620
|
+
if self._is_ios():
|
|
3621
|
+
return {"success": False, "reason": "iOS 请使用 _close_popup_ios"}
|
|
2988
3622
|
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
3623
|
+
if confidence_threshold is None:
|
|
3624
|
+
confidence_threshold = DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3625
|
+
|
|
3626
|
+
# 1. 获取元素列表(复用或新获取)
|
|
3627
|
+
if elements is None:
|
|
3628
|
+
elements = self.list_elements()
|
|
3629
|
+
|
|
3630
|
+
# 2. 分析页面内容(传入预获取的 XML)
|
|
3631
|
+
page_analysis = self._detect_page_content(
|
|
3632
|
+
elements,
|
|
3633
|
+
confidence_threshold=confidence_threshold,
|
|
3634
|
+
xml_string=xml_string
|
|
2999
3635
|
)
|
|
3000
3636
|
|
|
3001
|
-
#
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3637
|
+
# 3. 如果没有弹窗,直接返回
|
|
3638
|
+
if not page_analysis.get("has_popup"):
|
|
3639
|
+
return {"success": False, "reason": "未检测到弹窗", "method": "elements"}
|
|
3640
|
+
|
|
3641
|
+
# 4. 查找关闭按钮
|
|
3642
|
+
close_button = self._find_close_button_in_elements(elements, page_analysis)
|
|
3643
|
+
|
|
3644
|
+
# 5. 如果找到,点击
|
|
3645
|
+
if close_button:
|
|
3646
|
+
center_x = close_button['center_x']
|
|
3647
|
+
center_y = close_button['center_y']
|
|
3648
|
+
|
|
3649
|
+
self.client.u2.click(center_x, center_y)
|
|
3650
|
+
time.sleep(0.5)
|
|
3651
|
+
|
|
3652
|
+
# 检查应用是否跳转
|
|
3653
|
+
app_check = self._check_app_switched()
|
|
3654
|
+
return_result = None
|
|
3655
|
+
if app_check['switched']:
|
|
3656
|
+
return_result = self._return_to_target_app()
|
|
3657
|
+
|
|
3658
|
+
# 【优化】使用页面指纹对比验证
|
|
3659
|
+
verified = self._verify_popup_closed_with_fingerprint(page_fingerprint_before)
|
|
3660
|
+
|
|
3661
|
+
msg = f"✅ 通过控件树找到关闭按钮并点击\n"
|
|
3662
|
+
msg += f" 原因: {close_button['reason']}\n"
|
|
3663
|
+
msg += f" 位置: ({center_x}, {center_y}) [({close_button['x_percent']}%, {close_button['y_percent']}%)]"
|
|
3664
|
+
if close_button.get('text'):
|
|
3665
|
+
msg += f"\n 文本: {close_button['text']}"
|
|
3666
|
+
if close_button.get('resource_id'):
|
|
3667
|
+
msg += f"\n ID: {close_button['resource_id']}"
|
|
3668
|
+
|
|
3669
|
+
if app_check['switched']:
|
|
3670
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
3671
|
+
if return_result and return_result.get('success'):
|
|
3007
3672
|
msg += f"\n{return_result['message']}"
|
|
3673
|
+
|
|
3674
|
+
if not verified:
|
|
3675
|
+
msg += f"\n⚠️ 验证失败:弹窗可能未完全关闭"
|
|
3676
|
+
|
|
3677
|
+
return {
|
|
3678
|
+
"success": verified,
|
|
3679
|
+
"method": "elements",
|
|
3680
|
+
"message": msg,
|
|
3681
|
+
"clicked": {
|
|
3682
|
+
"center": (center_x, center_y),
|
|
3683
|
+
"percent": (close_button['x_percent'], close_button['y_percent']),
|
|
3684
|
+
"reason": close_button['reason']
|
|
3685
|
+
},
|
|
3686
|
+
"app_check": app_check,
|
|
3687
|
+
"return_to_app": return_result,
|
|
3688
|
+
"verified": verified
|
|
3689
|
+
}
|
|
3690
|
+
|
|
3691
|
+
return {"success": False, "reason": "控件树中未找到关闭按钮", "method": "elements"}
|
|
3692
|
+
|
|
3693
|
+
except Exception as e:
|
|
3694
|
+
return {"success": False, "reason": f"控件树方法失败: {e}", "method": "elements"}
|
|
3695
|
+
|
|
3696
|
+
def _close_popup_by_screenshot_ai(self, elements: List[Dict] = None) -> Dict:
|
|
3697
|
+
"""阶段2:通过截图AI分析关闭弹窗(仅在控件树失败时)
|
|
3698
|
+
|
|
3699
|
+
【优化】提供关闭按钮候选建议,形成闭环
|
|
3700
|
+
|
|
3701
|
+
Args:
|
|
3702
|
+
elements: 预获取的元素列表(可选)
|
|
3703
|
+
|
|
3704
|
+
Returns:
|
|
3705
|
+
关闭结果(包含候选建议,需要AI分析后确定最终操作)
|
|
3706
|
+
"""
|
|
3707
|
+
try:
|
|
3708
|
+
if self._is_ios():
|
|
3709
|
+
return {"success": False, "reason": "iOS 暂不支持"}
|
|
3710
|
+
|
|
3711
|
+
# 截图(带SoM标注)
|
|
3712
|
+
som_result = self.take_screenshot_with_som()
|
|
3713
|
+
|
|
3714
|
+
# 【优化】基于元素列表提供关闭按钮候选建议
|
|
3715
|
+
close_button_candidates = []
|
|
3716
|
+
if elements is None:
|
|
3717
|
+
elements = self.list_elements()
|
|
3718
|
+
|
|
3719
|
+
# 查找所有可能的关闭按钮
|
|
3720
|
+
for elem in elements:
|
|
3721
|
+
text = elem.get("text", "").strip()
|
|
3722
|
+
resource_id = elem.get("resource_id", "").strip()
|
|
3723
|
+
content_desc = elem.get("content_desc", "").strip()
|
|
3724
|
+
bounds = elem.get("bounds", "")
|
|
3725
|
+
|
|
3726
|
+
if not bounds:
|
|
3727
|
+
continue
|
|
3728
|
+
|
|
3729
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
3730
|
+
if not match:
|
|
3731
|
+
continue
|
|
3732
|
+
|
|
3733
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
3734
|
+
center_x = (x1 + x2) // 2
|
|
3735
|
+
center_y = (y1 + y2) // 2
|
|
3736
|
+
|
|
3737
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3738
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3739
|
+
x_percent = round(center_x / screen_width * 100, 1)
|
|
3740
|
+
y_percent = round(center_y / screen_height * 100, 1)
|
|
3741
|
+
|
|
3742
|
+
score = 0
|
|
3743
|
+
reason = ""
|
|
3744
|
+
|
|
3745
|
+
# 检查是否是确认按钮(优先)
|
|
3746
|
+
if text in CONFIRM_BUTTON_TEXTS:
|
|
3747
|
+
score = 120
|
|
3748
|
+
reason = f"确认按钮文本='{text}'"
|
|
3749
|
+
elif any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3750
|
+
score = 115
|
|
3751
|
+
reason = f"确认按钮ID: {resource_id}"
|
|
3752
|
+
elif any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3753
|
+
score = 110
|
|
3754
|
+
reason = f"确认按钮描述: {content_desc}"
|
|
3755
|
+
# 检查是否是关闭按钮(次优先)
|
|
3756
|
+
elif text in CLOSE_BUTTON_TEXTS:
|
|
3757
|
+
score = 100
|
|
3758
|
+
reason = f"关闭按钮文本='{text}'"
|
|
3759
|
+
elif any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3760
|
+
score = 95
|
|
3761
|
+
reason = f"关闭按钮ID: {resource_id}"
|
|
3762
|
+
elif any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3763
|
+
score = 90
|
|
3764
|
+
reason = f"关闭按钮描述: {content_desc}"
|
|
3765
|
+
|
|
3766
|
+
if score > 0:
|
|
3767
|
+
# 确定推荐操作方式
|
|
3768
|
+
if text in CONFIRM_BUTTON_TEXTS or text in CLOSE_BUTTON_TEXTS:
|
|
3769
|
+
suggested_action = f"mobile_click_by_text('{text}')"
|
|
3008
3770
|
else:
|
|
3009
|
-
|
|
3771
|
+
suggested_action = f"mobile_click_by_percent({x_percent}, {y_percent})"
|
|
3772
|
+
|
|
3773
|
+
close_button_candidates.append({
|
|
3774
|
+
'score': score,
|
|
3775
|
+
'reason': reason,
|
|
3776
|
+
'text': text,
|
|
3777
|
+
'resource_id': resource_id,
|
|
3778
|
+
'center_x': center_x,
|
|
3779
|
+
'center_y': center_y,
|
|
3780
|
+
'x_percent': x_percent,
|
|
3781
|
+
'y_percent': y_percent,
|
|
3782
|
+
'suggested_action': suggested_action
|
|
3783
|
+
})
|
|
3784
|
+
|
|
3785
|
+
# 按分数排序
|
|
3786
|
+
close_button_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3010
3787
|
|
|
3011
|
-
#
|
|
3012
|
-
# 如果弹窗还在,AI 可以选择点击其他候选按钮
|
|
3788
|
+
# 返回结果,包含候选建议
|
|
3013
3789
|
return {
|
|
3014
|
-
"success":
|
|
3015
|
-
"
|
|
3016
|
-
"
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3023
|
-
"popup_detected": popup_detected,
|
|
3024
|
-
"popup_confidence": popup_confidence if popup_bounds else 0,
|
|
3025
|
-
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_detected else None,
|
|
3026
|
-
"app_check": app_check,
|
|
3027
|
-
"return_to_app": return_result,
|
|
3028
|
-
"other_candidates": [
|
|
3029
|
-
{
|
|
3030
|
-
"position": c['position'],
|
|
3031
|
-
"type": c['match_type'],
|
|
3032
|
-
"coords": (c['center_x'], c['center_y']),
|
|
3033
|
-
"percent": (c['x_percent'], c['y_percent'])
|
|
3034
|
-
}
|
|
3035
|
-
for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
|
|
3036
|
-
],
|
|
3037
|
-
"tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置"
|
|
3790
|
+
"success": False, # 需要AI分析后才能确定
|
|
3791
|
+
"method": "screenshot_ai",
|
|
3792
|
+
"message": "已截图,等待AI分析",
|
|
3793
|
+
"screenshot": som_result.get("screenshot_path"),
|
|
3794
|
+
"elements": som_result.get("elements", []),
|
|
3795
|
+
"popup_detected": som_result.get("popup_detected", False),
|
|
3796
|
+
"close_button_candidates": close_button_candidates[:5], # 返回前5个候选
|
|
3797
|
+
"next_step": "AI分析截图后,根据分析结果调用对应的点击方法"
|
|
3038
3798
|
}
|
|
3039
3799
|
|
|
3040
3800
|
except Exception as e:
|
|
3041
|
-
return {"success": False, "
|
|
3801
|
+
return {"success": False, "reason": f"截图AI方法失败: {e}", "method": "screenshot_ai"}
|
|
3802
|
+
|
|
3803
|
+
def _close_popup_by_template(self, page_fingerprint_before: Dict = None) -> Dict:
|
|
3804
|
+
"""阶段3:通过模板匹配关闭弹窗(最精确的兜底方案)
|
|
3805
|
+
|
|
3806
|
+
Args:
|
|
3807
|
+
page_fingerprint_before: 操作前的页面指纹(用于验证)
|
|
3808
|
+
|
|
3809
|
+
Returns:
|
|
3810
|
+
关闭结果
|
|
3811
|
+
"""
|
|
3812
|
+
try:
|
|
3813
|
+
if self._is_ios():
|
|
3814
|
+
return {"success": False, "reason": "iOS 暂不支持"}
|
|
3815
|
+
|
|
3816
|
+
# 模板匹配
|
|
3817
|
+
match_result = self.template_match_close()
|
|
3818
|
+
|
|
3819
|
+
# 如果找到匹配,点击
|
|
3820
|
+
if match_result.get("success") and match_result.get("matches"):
|
|
3821
|
+
best_match = match_result["matches"][0]
|
|
3822
|
+
x, y = best_match["center"]
|
|
3823
|
+
|
|
3824
|
+
self.client.u2.click(x, y)
|
|
3825
|
+
time.sleep(0.5)
|
|
3826
|
+
|
|
3827
|
+
# 检查应用是否跳转
|
|
3828
|
+
app_check = self._check_app_switched()
|
|
3829
|
+
return_result = None
|
|
3830
|
+
if app_check['switched']:
|
|
3831
|
+
return_result = self._return_to_target_app()
|
|
3832
|
+
|
|
3833
|
+
# 【优化】使用页面指纹对比验证
|
|
3834
|
+
verified = self._verify_popup_closed_with_fingerprint(page_fingerprint_before)
|
|
3835
|
+
|
|
3836
|
+
msg = f"✅ 通过模板匹配找到关闭按钮并点击\n"
|
|
3837
|
+
msg += f" 模板: {best_match.get('template', 'unknown')}\n"
|
|
3838
|
+
msg += f" 置信度: {best_match.get('confidence', 0):.1f}%\n"
|
|
3839
|
+
msg += f" 位置: ({x}, {y})"
|
|
3840
|
+
|
|
3841
|
+
if app_check['switched']:
|
|
3842
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
3843
|
+
if return_result and return_result.get('success'):
|
|
3844
|
+
msg += f"\n{return_result['message']}"
|
|
3845
|
+
|
|
3846
|
+
if not verified:
|
|
3847
|
+
msg += f"\n⚠️ 验证失败:弹窗可能未完全关闭"
|
|
3848
|
+
|
|
3849
|
+
return {
|
|
3850
|
+
"success": verified,
|
|
3851
|
+
"method": "template",
|
|
3852
|
+
"message": msg,
|
|
3853
|
+
"clicked": {"center": (x, y)},
|
|
3854
|
+
"app_check": app_check,
|
|
3855
|
+
"return_to_app": return_result,
|
|
3856
|
+
"verified": verified
|
|
3857
|
+
}
|
|
3858
|
+
|
|
3859
|
+
return {"success": False, "reason": "模板匹配未找到关闭按钮", "method": "template"}
|
|
3860
|
+
|
|
3861
|
+
except ImportError:
|
|
3862
|
+
return {"success": False, "reason": "需要安装 opencv-python: pip install opencv-python", "method": "template"}
|
|
3863
|
+
except Exception as e:
|
|
3864
|
+
return {"success": False, "reason": f"模板匹配失败: {e}", "method": "template"}
|
|
3865
|
+
|
|
3866
|
+
def _verify_popup_closed(self) -> bool:
|
|
3867
|
+
"""验证弹窗是否已关闭(简单版本)
|
|
3868
|
+
|
|
3869
|
+
Returns:
|
|
3870
|
+
True 如果弹窗已关闭,False 如果弹窗仍在
|
|
3871
|
+
"""
|
|
3872
|
+
try:
|
|
3873
|
+
time.sleep(0.5)
|
|
3874
|
+
elements = self.list_elements()
|
|
3875
|
+
page_analysis = self._detect_page_content(elements)
|
|
3876
|
+
return not page_analysis.get("has_popup")
|
|
3877
|
+
except Exception:
|
|
3878
|
+
return False
|
|
3879
|
+
|
|
3880
|
+
def _verify_popup_closed_with_fingerprint(self, fingerprint_before: Dict = None) -> bool:
|
|
3881
|
+
"""验证弹窗是否已关闭(使用页面指纹对比)
|
|
3882
|
+
|
|
3883
|
+
Args:
|
|
3884
|
+
fingerprint_before: 操作前的页面指纹
|
|
3885
|
+
|
|
3886
|
+
Returns:
|
|
3887
|
+
True 如果弹窗已关闭,False 如果弹窗仍在
|
|
3888
|
+
"""
|
|
3889
|
+
try:
|
|
3890
|
+
time.sleep(0.5)
|
|
3891
|
+
|
|
3892
|
+
# 获取当前元素列表
|
|
3893
|
+
elements = self.list_elements()
|
|
3894
|
+
|
|
3895
|
+
# 方法1:检查是否还有弹窗
|
|
3896
|
+
page_analysis = self._detect_page_content(elements)
|
|
3897
|
+
if not page_analysis.get("has_popup"):
|
|
3898
|
+
return True
|
|
3899
|
+
|
|
3900
|
+
# 方法2:如果有操作前的指纹,对比页面变化
|
|
3901
|
+
if fingerprint_before:
|
|
3902
|
+
fingerprint_after = self._get_page_fingerprint(elements)
|
|
3903
|
+
page_changed = self._compare_page_fingerprint(fingerprint_before, fingerprint_after)
|
|
3904
|
+
|
|
3905
|
+
# 如果页面发生了明显变化,认为弹窗可能已关闭
|
|
3906
|
+
# (即使检测到还有弹窗特征,也可能是新弹窗或检测误判)
|
|
3907
|
+
if page_changed:
|
|
3908
|
+
return True
|
|
3909
|
+
|
|
3910
|
+
return False
|
|
3911
|
+
|
|
3912
|
+
except Exception:
|
|
3913
|
+
return False
|
|
3042
3914
|
|
|
3043
3915
|
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
3044
3916
|
"""根据相对坐标获取位置名称"""
|
|
@@ -3082,9 +3954,12 @@ class BasicMobileToolsLite:
|
|
|
3082
3954
|
else: # 中间区域
|
|
3083
3955
|
return 0.5
|
|
3084
3956
|
|
|
3085
|
-
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) ->
|
|
3957
|
+
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) -> Tuple[Optional[Tuple[int, int, int, int]], float]:
|
|
3086
3958
|
"""严格的弹窗检测 - 使用置信度评分,避免误识别普通页面
|
|
3087
3959
|
|
|
3960
|
+
使用全局常量 POPUP_CLASS_KEYWORDS、POPUP_ID_KEYWORDS、AD_POPUP_KEYWORDS、
|
|
3961
|
+
PAGE_CONTENT_KEYWORDS、CLOSE_BUTTON_KEYWORDS
|
|
3962
|
+
|
|
3088
3963
|
真正的弹窗特征:
|
|
3089
3964
|
1. class 名称包含 Dialog/Popup/Alert/Modal/BottomSheet(强特征)
|
|
3090
3965
|
2. resource-id 包含 dialog/popup/alert/modal(强特征)
|
|
@@ -3094,10 +3969,8 @@ class BasicMobileToolsLite:
|
|
|
3094
3969
|
|
|
3095
3970
|
Returns:
|
|
3096
3971
|
(popup_bounds, confidence) 或 (None, 0)
|
|
3097
|
-
confidence >=
|
|
3972
|
+
confidence >= DEFAULT_POPUP_CONFIDENCE_THRESHOLD 才认为是弹窗
|
|
3098
3973
|
"""
|
|
3099
|
-
import re
|
|
3100
|
-
|
|
3101
3974
|
screen_area = screen_width * screen_height
|
|
3102
3975
|
|
|
3103
3976
|
# 收集所有元素信息
|
|
@@ -3119,14 +3992,14 @@ class BasicMobileToolsLite:
|
|
|
3119
3992
|
class_name = elem.attrib.get('class', '')
|
|
3120
3993
|
resource_id = elem.attrib.get('resource-id', '')
|
|
3121
3994
|
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
3995
|
+
text = elem.attrib.get('text', '')
|
|
3122
3996
|
|
|
3123
|
-
#
|
|
3997
|
+
# 检查是否是关闭/确认按钮 - 使用全局常量
|
|
3124
3998
|
is_close_button = (
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
'X' in elem.attrib.get('text', '')
|
|
3999
|
+
any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS) or
|
|
4000
|
+
any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS) or
|
|
4001
|
+
text in CLOSE_BUTTON_TEXTS or
|
|
4002
|
+
text in CONFIRM_BUTTON_TEXTS
|
|
3130
4003
|
)
|
|
3131
4004
|
|
|
3132
4005
|
all_elements.append({
|
|
@@ -3142,17 +4015,12 @@ class BasicMobileToolsLite:
|
|
|
3142
4015
|
'center_x': (x1 + x2) // 2,
|
|
3143
4016
|
'center_y': (y1 + y2) // 2,
|
|
3144
4017
|
'is_close_button': is_close_button,
|
|
4018
|
+
'text': text,
|
|
3145
4019
|
})
|
|
3146
4020
|
|
|
3147
4021
|
if not all_elements:
|
|
3148
4022
|
return None, 0
|
|
3149
4023
|
|
|
3150
|
-
# 弹窗检测关键词
|
|
3151
|
-
dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
|
|
3152
|
-
dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
|
|
3153
|
-
# 广告弹窗关键词(全屏广告、激励视频等)
|
|
3154
|
-
ad_popup_keywords = ['ad_close', 'ad_button', 'full_screen', 'interstitial', 'reward', 'close_icon', 'close_btn']
|
|
3155
|
-
|
|
3156
4024
|
popup_candidates = []
|
|
3157
4025
|
has_mask_layer = False
|
|
3158
4026
|
mask_idx = -1
|
|
@@ -3183,22 +4051,20 @@ class BasicMobileToolsLite:
|
|
|
3183
4051
|
continue
|
|
3184
4052
|
|
|
3185
4053
|
# 【非弹窗特征】如果元素包含底部导航栏(底部tab),则不是弹窗
|
|
3186
|
-
# 底部导航栏通常在屏幕底部,高度约100-200像素
|
|
3187
4054
|
if y2 > screen_height * 0.85:
|
|
3188
|
-
# 检查是否包含tab相关的resource-id或class
|
|
3189
4055
|
if 'tab' in resource_id.lower() or 'Tab' in class_name or 'navigation' in resource_id.lower():
|
|
3190
|
-
continue
|
|
4056
|
+
continue
|
|
3191
4057
|
|
|
3192
4058
|
# 【非弹窗特征】如果元素包含顶部搜索栏,则不是弹窗
|
|
3193
4059
|
if y1 < screen_height * 0.15:
|
|
3194
4060
|
if 'search' in resource_id.lower() or 'Search' in class_name:
|
|
3195
|
-
continue
|
|
4061
|
+
continue
|
|
3196
4062
|
|
|
3197
|
-
#
|
|
4063
|
+
# 使用全局常量检查是否有强弹窗特征
|
|
3198
4064
|
has_strong_popup_feature = (
|
|
3199
|
-
any(kw in class_name for kw in
|
|
3200
|
-
any(kw in resource_id.lower() for kw in
|
|
3201
|
-
any(kw in resource_id.lower() for kw in
|
|
4065
|
+
any(kw in class_name for kw in POPUP_CLASS_KEYWORDS) or
|
|
4066
|
+
any(kw in resource_id.lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4067
|
+
any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS)
|
|
3202
4068
|
)
|
|
3203
4069
|
|
|
3204
4070
|
# 检查是否有子元素是关闭按钮(作为弹窗特征)
|
|
@@ -3208,45 +4074,37 @@ class BasicMobileToolsLite:
|
|
|
3208
4074
|
if other_elem['idx'] == elem['idx']:
|
|
3209
4075
|
continue
|
|
3210
4076
|
if other_elem['is_close_button']:
|
|
3211
|
-
# 检查关闭按钮是否在这个元素范围内
|
|
3212
4077
|
ox1, oy1, ox2, oy2 = other_elem['bounds']
|
|
3213
4078
|
ex1, ey1, ex2, ey2 = elem_bounds
|
|
3214
4079
|
if ex1 <= ox1 and ey1 <= oy1 and ex2 >= ox2 and ey2 >= oy2:
|
|
3215
4080
|
has_close_button_child = True
|
|
3216
4081
|
break
|
|
3217
4082
|
|
|
3218
|
-
#
|
|
3219
|
-
|
|
3220
|
-
page_content_keywords = ['video', 'player', 'recycler', 'list', 'scroll', 'viewpager', 'fragment']
|
|
3221
|
-
if any(kw in resource_id.lower() or kw in class_name.lower() for kw in page_content_keywords):
|
|
3222
|
-
# 如果面积很大且没有强弹窗特征,则不是弹窗
|
|
4083
|
+
# 【非弹窗特征】使用全局常量检查页面内容特征
|
|
4084
|
+
if any(kw in resource_id.lower() or kw in class_name.lower() for kw in PAGE_CONTENT_KEYWORDS):
|
|
3223
4085
|
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
3224
4086
|
continue
|
|
3225
4087
|
|
|
3226
|
-
#
|
|
3227
|
-
# 真正的弹窗通常不会超过屏幕的60%
|
|
3228
|
-
# 对于面积 > 0.6 的元素,如果没有强特征,直接跳过(避免误判首页内容区域)
|
|
4088
|
+
# 【非弹窗特征】面积过大且无强特征
|
|
3229
4089
|
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
3230
|
-
continue
|
|
4090
|
+
continue
|
|
3231
4091
|
|
|
3232
|
-
# 对于面积 > 0.7 的元素,即使有强特征也要更严格
|
|
3233
4092
|
if area_ratio > 0.7:
|
|
3234
|
-
# 需要非常强的特征才认为是弹窗
|
|
3235
4093
|
if not has_strong_popup_feature:
|
|
3236
4094
|
continue
|
|
3237
4095
|
|
|
3238
4096
|
confidence = 0.0
|
|
3239
4097
|
|
|
3240
|
-
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
3241
|
-
if any(kw in class_name for kw in
|
|
4098
|
+
# 【强特征】class 名称包含弹窗关键词 (+0.5) - 使用全局常量
|
|
4099
|
+
if any(kw in class_name for kw in POPUP_CLASS_KEYWORDS):
|
|
3242
4100
|
confidence += 0.5
|
|
3243
4101
|
|
|
3244
|
-
# 【强特征】resource-id 包含弹窗关键词 (+0.4)
|
|
3245
|
-
if any(kw in resource_id.lower() for kw in
|
|
4102
|
+
# 【强特征】resource-id 包含弹窗关键词 (+0.4) - 使用全局常量
|
|
4103
|
+
if any(kw in resource_id.lower() for kw in POPUP_ID_KEYWORDS):
|
|
3246
4104
|
confidence += 0.4
|
|
3247
4105
|
|
|
3248
|
-
# 【强特征】resource-id 包含广告弹窗关键词 (+0.4)
|
|
3249
|
-
if any(kw in resource_id.lower() for kw in
|
|
4106
|
+
# 【强特征】resource-id 包含广告弹窗关键词 (+0.4) - 使用全局常量
|
|
4107
|
+
if any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS):
|
|
3250
4108
|
confidence += 0.4
|
|
3251
4109
|
|
|
3252
4110
|
# 【强特征】包含关闭按钮作为子元素 (+0.3)
|
|
@@ -3254,37 +4112,26 @@ class BasicMobileToolsLite:
|
|
|
3254
4112
|
confidence += 0.3
|
|
3255
4113
|
|
|
3256
4114
|
# 【中等特征】居中显示 (+0.2)
|
|
3257
|
-
# 但如果没有强特征,降低权重
|
|
3258
4115
|
center_x = elem['center_x']
|
|
3259
4116
|
center_y = elem['center_y']
|
|
3260
4117
|
is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
|
|
3261
4118
|
is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
|
|
3262
4119
|
|
|
3263
4120
|
has_strong_feature = (
|
|
3264
|
-
any(kw in class_name for kw in
|
|
3265
|
-
any(kw in resource_id.lower() for kw in
|
|
3266
|
-
any(kw in resource_id.lower() for kw in
|
|
4121
|
+
any(kw in class_name for kw in POPUP_CLASS_KEYWORDS) or
|
|
4122
|
+
any(kw in resource_id.lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4123
|
+
any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS) or
|
|
3267
4124
|
has_close_button_child
|
|
3268
4125
|
)
|
|
3269
4126
|
|
|
3270
4127
|
if is_centered_x and is_centered_y:
|
|
3271
|
-
if has_strong_feature
|
|
3272
|
-
confidence += 0.2
|
|
3273
|
-
else:
|
|
3274
|
-
confidence += 0.1 # 没有强特征时降低权重
|
|
4128
|
+
confidence += 0.2 if has_strong_feature else 0.1
|
|
3275
4129
|
elif is_centered_x:
|
|
3276
|
-
if has_strong_feature
|
|
3277
|
-
confidence += 0.1
|
|
3278
|
-
else:
|
|
3279
|
-
confidence += 0.05 # 没有强特征时降低权重
|
|
4130
|
+
confidence += 0.1 if has_strong_feature else 0.05
|
|
3280
4131
|
|
|
3281
4132
|
# 【中等特征】非全屏但有一定大小 (+0.15)
|
|
3282
|
-
# 但如果没有强特征,降低权重
|
|
3283
4133
|
if 0.15 < area_ratio < 0.75:
|
|
3284
|
-
if has_strong_feature
|
|
3285
|
-
confidence += 0.15
|
|
3286
|
-
else:
|
|
3287
|
-
confidence += 0.08 # 没有强特征时降低权重
|
|
4134
|
+
confidence += 0.15 if has_strong_feature else 0.08
|
|
3288
4135
|
|
|
3289
4136
|
# 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
|
|
3290
4137
|
if elem['idx'] > len(all_elements) * 0.5:
|
|
@@ -3311,20 +4158,15 @@ class BasicMobileToolsLite:
|
|
|
3311
4158
|
popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
|
|
3312
4159
|
best = popup_candidates[0]
|
|
3313
4160
|
|
|
3314
|
-
#
|
|
3315
|
-
# 如果没有强特征(class或resource-id包含弹窗关键词),需要更高的置信度
|
|
4161
|
+
# 更严格的阈值判断 - 使用全局常量
|
|
3316
4162
|
has_strong_feature = (
|
|
3317
|
-
any(kw in best['class'] for kw in
|
|
3318
|
-
any(kw in best['resource_id'].lower() for kw in
|
|
3319
|
-
any(kw in best['resource_id'].lower() for kw in
|
|
4163
|
+
any(kw in best['class'] for kw in POPUP_CLASS_KEYWORDS) or
|
|
4164
|
+
any(kw in best['resource_id'].lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4165
|
+
any(kw in best['resource_id'].lower() for kw in AD_POPUP_KEYWORDS)
|
|
3320
4166
|
)
|
|
3321
4167
|
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
threshold = 0.7
|
|
3325
|
-
else:
|
|
3326
|
-
# 没有强特征时,阈值0.85(更严格)
|
|
3327
|
-
threshold = 0.85
|
|
4168
|
+
# 有强特征时阈值 0.7,否则 0.85
|
|
4169
|
+
threshold = 0.7 if has_strong_feature else 0.85
|
|
3328
4170
|
|
|
3329
4171
|
if best['confidence'] >= threshold:
|
|
3330
4172
|
return best['bounds'], best['confidence']
|
|
@@ -4011,239 +4853,229 @@ class BasicMobileToolsLite:
|
|
|
4011
4853
|
except Exception as e:
|
|
4012
4854
|
return {"success": False, "error": f"删除模板失败: {e}"}
|
|
4013
4855
|
|
|
4014
|
-
def
|
|
4015
|
-
"""
|
|
4856
|
+
def close_popup(self, auto_learn: bool = False, confidence_threshold: float = None) -> Dict:
|
|
4857
|
+
"""智能关闭弹窗(优化版)
|
|
4858
|
+
|
|
4859
|
+
优化后的策略(降级策略):
|
|
4860
|
+
1. 控件树优先(最快、最可靠):list_elements() → 查找关闭按钮 → 点击
|
|
4861
|
+
2. 截图AI分析(中等速度、高准确率):如果控件树失败 → 截图 → AI分析 → 返回候选
|
|
4862
|
+
3. 模板匹配(最慢、最精确):如果AI分析失败 → 模板匹配 → 点击
|
|
4016
4863
|
|
|
4017
|
-
|
|
4018
|
-
1. 控件树查找关闭按钮(最可靠)
|
|
4019
|
-
2. 模板匹配(需要积累模板库)
|
|
4020
|
-
3. 返回视觉信息供 AI 分析(如果前两步失败)
|
|
4864
|
+
每个阶段都会使用页面指纹对比验证弹窗是否真的关闭,如果验证失败会继续下一阶段。
|
|
4021
4865
|
|
|
4022
|
-
|
|
4023
|
-
-
|
|
4024
|
-
- 如果是新样式,自动裁剪并添加到模板库
|
|
4866
|
+
自动学习(可选):
|
|
4867
|
+
- 如果通过模板匹配成功关闭,且 auto_learn=True,会自动学习新模板
|
|
4025
4868
|
|
|
4026
4869
|
Args:
|
|
4027
|
-
auto_learn:
|
|
4870
|
+
auto_learn: 是否自动学习新模板(点击成功后检查并保存),默认 False
|
|
4871
|
+
confidence_threshold: 弹窗检测置信度阈值,默认使用 DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
4028
4872
|
|
|
4029
4873
|
Returns:
|
|
4030
|
-
|
|
4874
|
+
关闭结果
|
|
4031
4875
|
"""
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
result = {
|
|
4036
|
-
"success": False,
|
|
4037
|
-
"method": None,
|
|
4038
|
-
"message": "",
|
|
4039
|
-
"learned_template": None
|
|
4040
|
-
}
|
|
4041
|
-
|
|
4042
|
-
if self._is_ios():
|
|
4043
|
-
return {"success": False, "error": "iOS 暂不支持此功能"}
|
|
4876
|
+
if confidence_threshold is None:
|
|
4877
|
+
confidence_threshold = DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
4044
4878
|
|
|
4045
4879
|
try:
|
|
4046
|
-
|
|
4880
|
+
# iOS 也支持基础弹窗关闭
|
|
4881
|
+
if self._is_ios():
|
|
4882
|
+
return self._close_popup_ios()
|
|
4047
4883
|
|
|
4048
|
-
|
|
4049
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
4050
|
-
root = ET.fromstring(xml_string)
|
|
4884
|
+
tried_methods = []
|
|
4051
4885
|
|
|
4052
|
-
#
|
|
4053
|
-
|
|
4054
|
-
|
|
4886
|
+
# 【优化】一次性获取元素列表和 XML,复用于多个检测方法
|
|
4887
|
+
elements = self.list_elements()
|
|
4888
|
+
xml_string = None
|
|
4889
|
+
try:
|
|
4890
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
4891
|
+
except Exception:
|
|
4892
|
+
pass
|
|
4055
4893
|
|
|
4056
|
-
|
|
4894
|
+
# 保存页面指纹用于验证
|
|
4895
|
+
page_fingerprint_before = self._get_page_fingerprint(elements)
|
|
4057
4896
|
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
for kw in close_content_desc:
|
|
4089
|
-
if kw.lower() in content_desc.lower():
|
|
4090
|
-
score += 8
|
|
4091
|
-
reason = f"描述含'{kw}'"
|
|
4092
|
-
break
|
|
4093
|
-
|
|
4094
|
-
# 小尺寸可点击元素(可能是 X 按钮)
|
|
4095
|
-
if clickable and 30 < width < 200 and 30 < height < 200:
|
|
4096
|
-
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
4097
|
-
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
4098
|
-
|
|
4099
|
-
# 在屏幕右半边上半部分,很可能是 X
|
|
4100
|
-
if cx > screen_width * 0.6 and cy < screen_height * 0.5:
|
|
4101
|
-
score += 5
|
|
4102
|
-
reason = reason or "右上角小按钮"
|
|
4103
|
-
# 在屏幕上半部分的小按钮,也可能是 X
|
|
4104
|
-
elif cy < screen_height * 0.4:
|
|
4105
|
-
score += 2
|
|
4106
|
-
reason = reason or "上部小按钮"
|
|
4897
|
+
# 阶段1:控件树优先(传入预获取的元素和XML)
|
|
4898
|
+
result = self._close_popup_by_elements(
|
|
4899
|
+
elements=elements,
|
|
4900
|
+
xml_string=xml_string,
|
|
4901
|
+
confidence_threshold=confidence_threshold,
|
|
4902
|
+
page_fingerprint_before=page_fingerprint_before
|
|
4903
|
+
)
|
|
4904
|
+
tried_methods.append("elements")
|
|
4905
|
+
if result.get("success") and result.get("verified", False):
|
|
4906
|
+
return result
|
|
4907
|
+
# 如果明确检测到没有弹窗,直接返回,不继续后续阶段
|
|
4908
|
+
if result.get("reason") == "未检测到弹窗":
|
|
4909
|
+
return {
|
|
4910
|
+
"success": False,
|
|
4911
|
+
"message": "✅ 未检测到弹窗,无需关闭",
|
|
4912
|
+
"method": "elements",
|
|
4913
|
+
"tried_methods": tried_methods
|
|
4914
|
+
}
|
|
4915
|
+
|
|
4916
|
+
# 阶段2:截图AI分析(优化:提供候选建议)
|
|
4917
|
+
result = self._close_popup_by_screenshot_ai(elements=elements)
|
|
4918
|
+
tried_methods.append("screenshot_ai")
|
|
4919
|
+
if result.get("screenshot"):
|
|
4920
|
+
# 增加候选按钮建议
|
|
4921
|
+
candidates = result.get("close_button_candidates", [])
|
|
4922
|
+
tip = "请AI分析截图,找到关闭按钮后调用 mobile_click_by_text/click_by_id/click_by_percent 等方法"
|
|
4923
|
+
if candidates:
|
|
4924
|
+
tip += f"\n💡 候选关闭按钮(按优先级):"
|
|
4925
|
+
for i, c in enumerate(candidates[:3], 1):
|
|
4926
|
+
tip += f"\n {i}. {c.get('reason', '')} at ({c.get('x_percent', 0)}%, {c.get('y_percent', 0)}%)"
|
|
4107
4927
|
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
close_candidates.append({
|
|
4114
|
-
'score': score,
|
|
4115
|
-
'reason': reason,
|
|
4116
|
-
'bounds': (x1, y1, x2, y2),
|
|
4117
|
-
'center': (cx, cy),
|
|
4118
|
-
'resource_id': resource_id,
|
|
4119
|
-
'text': text
|
|
4120
|
-
})
|
|
4928
|
+
return {
|
|
4929
|
+
**result,
|
|
4930
|
+
"tried_methods": tried_methods,
|
|
4931
|
+
"tip": tip
|
|
4932
|
+
}
|
|
4121
4933
|
|
|
4122
|
-
#
|
|
4123
|
-
|
|
4934
|
+
# 阶段3:模板匹配
|
|
4935
|
+
pre_screenshot_path = None
|
|
4936
|
+
if auto_learn:
|
|
4937
|
+
try:
|
|
4938
|
+
pre_screenshot_result = self.take_screenshot(description="关闭前(用于自动学习)", compress=False)
|
|
4939
|
+
pre_screenshot_path = pre_screenshot_result.get("screenshot_path")
|
|
4940
|
+
except Exception:
|
|
4941
|
+
pass
|
|
4124
4942
|
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
4141
|
-
app_check = self._check_app_switched()
|
|
4142
|
-
return_result = None
|
|
4143
|
-
|
|
4144
|
-
if app_check['switched']:
|
|
4145
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
4146
|
-
return_result = self._return_to_target_app()
|
|
4147
|
-
|
|
4148
|
-
result["success"] = True
|
|
4149
|
-
result["method"] = "控件树"
|
|
4150
|
-
msg = f"✅ 通过控件树找到关闭按钮并点击\n" \
|
|
4151
|
-
f" 位置: ({cx}, {cy})\n" \
|
|
4152
|
-
f" 原因: {best['reason']}"
|
|
4153
|
-
|
|
4154
|
-
if app_check['switched']:
|
|
4155
|
-
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
4156
|
-
if return_result:
|
|
4157
|
-
if return_result['success']:
|
|
4158
|
-
msg += f"\n{return_result['message']}"
|
|
4159
|
-
else:
|
|
4160
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
4161
|
-
|
|
4162
|
-
result["message"] = msg
|
|
4163
|
-
result["app_check"] = app_check
|
|
4164
|
-
result["return_to_app"] = return_result
|
|
4165
|
-
|
|
4166
|
-
# 自动学习:检查这个 X 是否已在模板库,不在就添加
|
|
4167
|
-
if auto_learn and pre_screenshot:
|
|
4168
|
-
learn_result = self._auto_learn_template(pre_screenshot, bounds)
|
|
4169
|
-
if learn_result:
|
|
4170
|
-
result["learned_template"] = learn_result
|
|
4171
|
-
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
4943
|
+
result = self._close_popup_by_template(page_fingerprint_before=page_fingerprint_before)
|
|
4944
|
+
tried_methods.append("template")
|
|
4945
|
+
if result.get("success") and result.get("verified", False):
|
|
4946
|
+
if auto_learn and result.get("method") == "template" and pre_screenshot_path:
|
|
4947
|
+
clicked = result.get("clicked", {})
|
|
4948
|
+
if clicked and "center" in clicked:
|
|
4949
|
+
x, y = clicked["center"]
|
|
4950
|
+
bounds = (x - 30, y - 30, x + 30, y + 30)
|
|
4951
|
+
try:
|
|
4952
|
+
learn_result = self._auto_learn_template(pre_screenshot_path, bounds)
|
|
4953
|
+
if learn_result:
|
|
4954
|
+
result["learned_template"] = learn_result
|
|
4955
|
+
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
4956
|
+
except Exception:
|
|
4957
|
+
pass
|
|
4172
4958
|
|
|
4173
4959
|
return result
|
|
4174
4960
|
|
|
4175
|
-
#
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4961
|
+
# 所有方法都失败
|
|
4962
|
+
return {
|
|
4963
|
+
"success": False,
|
|
4964
|
+
"message": "❌ 所有关闭弹窗方法都失败",
|
|
4965
|
+
"tried_methods": tried_methods,
|
|
4966
|
+
"suggestion": "请手动查看截图,找到关闭按钮位置,然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
4967
|
+
}
|
|
4968
|
+
|
|
4969
|
+
except Exception as e:
|
|
4970
|
+
return {"success": False, "message": f"❌ 关闭弹窗失败: {e}"}
|
|
4971
|
+
|
|
4972
|
+
def _close_popup_ios(self) -> Dict:
|
|
4973
|
+
"""iOS 平台关闭弹窗
|
|
4974
|
+
|
|
4975
|
+
Returns:
|
|
4976
|
+
关闭结果
|
|
4977
|
+
"""
|
|
4978
|
+
try:
|
|
4979
|
+
elements = self.list_elements()
|
|
4980
|
+
page_analysis = self._detect_page_content_ios(elements, DEFAULT_POPUP_CONFIDENCE_THRESHOLD)
|
|
4981
|
+
|
|
4982
|
+
if not page_analysis.get("has_popup"):
|
|
4983
|
+
return {"success": False, "reason": "未检测到弹窗", "method": "ios_elements"}
|
|
4984
|
+
|
|
4985
|
+
close_button = self._find_close_button_in_elements_ios(elements)
|
|
4986
|
+
|
|
4987
|
+
if close_button:
|
|
4988
|
+
ios_client = self._get_ios_client()
|
|
4989
|
+
if ios_client:
|
|
4990
|
+
ios_client.click(close_button['center_x'], close_button['center_y'])
|
|
4991
|
+
time.sleep(0.5)
|
|
4187
4992
|
|
|
4188
|
-
#
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
y_pct = best["percent"]["y"]
|
|
4193
|
-
|
|
4194
|
-
# 点击(click_by_percent 内部已包含应用状态检查和自动返回)
|
|
4195
|
-
click_result = self.click_by_percent(x_pct, y_pct)
|
|
4196
|
-
time.sleep(0.5)
|
|
4197
|
-
|
|
4198
|
-
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
4199
|
-
app_check = self._check_app_switched()
|
|
4200
|
-
return_result = None
|
|
4201
|
-
|
|
4202
|
-
if app_check['switched']:
|
|
4203
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
4204
|
-
return_result = self._return_to_target_app()
|
|
4205
|
-
|
|
4206
|
-
result["success"] = True
|
|
4207
|
-
result["method"] = "模板匹配"
|
|
4208
|
-
msg = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
|
|
4209
|
-
f" 模板: {best.get('template', 'unknown')}\n" \
|
|
4210
|
-
f" 置信度: {best.get('confidence', 'N/A')}%\n" \
|
|
4211
|
-
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
4212
|
-
|
|
4213
|
-
if app_check['switched']:
|
|
4214
|
-
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
4215
|
-
if return_result:
|
|
4216
|
-
if return_result['success']:
|
|
4217
|
-
msg += f"\n{return_result['message']}"
|
|
4218
|
-
else:
|
|
4219
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
4220
|
-
|
|
4221
|
-
result["message"] = msg
|
|
4222
|
-
result["app_check"] = app_check
|
|
4223
|
-
result["return_to_app"] = return_result
|
|
4224
|
-
return result
|
|
4993
|
+
# 验证
|
|
4994
|
+
new_elements = self.list_elements()
|
|
4995
|
+
new_analysis = self._detect_page_content_ios(new_elements, DEFAULT_POPUP_CONFIDENCE_THRESHOLD)
|
|
4996
|
+
verified = not new_analysis.get("has_popup")
|
|
4225
4997
|
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4998
|
+
return {
|
|
4999
|
+
"success": verified,
|
|
5000
|
+
"method": "ios_elements",
|
|
5001
|
+
"message": f"✅ iOS 关闭弹窗: {close_button['reason']}",
|
|
5002
|
+
"clicked": {
|
|
5003
|
+
"center": (close_button['center_x'], close_button['center_y']),
|
|
5004
|
+
"percent": (close_button['x_percent'], close_button['y_percent'])
|
|
5005
|
+
},
|
|
5006
|
+
"verified": verified
|
|
5007
|
+
}
|
|
4230
5008
|
|
|
4231
|
-
|
|
4232
|
-
if not screenshot_path:
|
|
4233
|
-
screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
|
|
5009
|
+
return {"success": False, "reason": "未找到关闭按钮", "method": "ios_elements"}
|
|
4234
5010
|
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
5011
|
+
except Exception as e:
|
|
5012
|
+
return {"success": False, "reason": f"iOS 关闭弹窗失败: {e}", "method": "ios_elements"}
|
|
5013
|
+
|
|
5014
|
+
def _get_page_fingerprint(self, elements: List[Dict]) -> Dict:
|
|
5015
|
+
"""获取页面指纹(用于验证弹窗是否关闭)
|
|
5016
|
+
|
|
5017
|
+
Args:
|
|
5018
|
+
elements: 元素列表
|
|
4242
5019
|
|
|
4243
|
-
|
|
5020
|
+
Returns:
|
|
5021
|
+
页面指纹信息
|
|
5022
|
+
"""
|
|
5023
|
+
# 提取关键特征
|
|
5024
|
+
resource_ids = set()
|
|
5025
|
+
texts = set()
|
|
5026
|
+
clickable_count = 0
|
|
5027
|
+
|
|
5028
|
+
for elem in elements:
|
|
5029
|
+
rid = elem.get("resource_id", "")
|
|
5030
|
+
if rid:
|
|
5031
|
+
resource_ids.add(rid)
|
|
5032
|
+
text = elem.get("text", "")
|
|
5033
|
+
if text and len(text) < 50: # 只记录短文本
|
|
5034
|
+
texts.add(text)
|
|
5035
|
+
if elem.get("clickable"):
|
|
5036
|
+
clickable_count += 1
|
|
5037
|
+
|
|
5038
|
+
return {
|
|
5039
|
+
"element_count": len(elements),
|
|
5040
|
+
"clickable_count": clickable_count,
|
|
5041
|
+
"resource_ids": resource_ids,
|
|
5042
|
+
"texts": texts,
|
|
5043
|
+
"resource_id_hash": hash(frozenset(resource_ids)) if resource_ids else 0,
|
|
5044
|
+
}
|
|
5045
|
+
|
|
5046
|
+
def _compare_page_fingerprint(self, before: Dict, after: Dict) -> bool:
|
|
5047
|
+
"""比较页面指纹,判断页面是否发生变化
|
|
5048
|
+
|
|
5049
|
+
Args:
|
|
5050
|
+
before: 操作前的页面指纹
|
|
5051
|
+
after: 操作后的页面指纹
|
|
4244
5052
|
|
|
4245
|
-
|
|
4246
|
-
|
|
5053
|
+
Returns:
|
|
5054
|
+
True 如果页面发生了明显变化(弹窗可能已关闭)
|
|
5055
|
+
"""
|
|
5056
|
+
if not before or not after:
|
|
5057
|
+
return False
|
|
5058
|
+
|
|
5059
|
+
# 元素数量变化
|
|
5060
|
+
count_diff = abs(after["element_count"] - before["element_count"])
|
|
5061
|
+
count_change_ratio = count_diff / max(before["element_count"], 1)
|
|
5062
|
+
|
|
5063
|
+
# resource_id 集合变化
|
|
5064
|
+
before_ids = before.get("resource_ids", set())
|
|
5065
|
+
after_ids = after.get("resource_ids", set())
|
|
5066
|
+
id_diff = len(before_ids.symmetric_difference(after_ids))
|
|
5067
|
+
id_change_ratio = id_diff / max(len(before_ids), 1)
|
|
5068
|
+
|
|
5069
|
+
# 如果元素数量或 resource_id 发生明显变化,认为页面已变化
|
|
5070
|
+
if count_change_ratio > 0.1 or id_change_ratio > 0.1:
|
|
5071
|
+
return True
|
|
5072
|
+
|
|
5073
|
+
# 如果 hash 值不同,也认为有变化
|
|
5074
|
+
if before.get("resource_id_hash") != after.get("resource_id_hash"):
|
|
5075
|
+
return True
|
|
5076
|
+
|
|
5077
|
+
return False
|
|
5078
|
+
|
|
4247
5079
|
|
|
4248
5080
|
def _detect_popup_region(self, root) -> tuple:
|
|
4249
5081
|
"""从控件树中检测弹窗区域
|