mobile-mcp-ai 2.6.10__py3-none-any.whl → 2.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/config.py +32 -0
- mobile_mcp/core/basic_tools_lite.py +852 -1843
- mobile_mcp/mcp_tools/mcp_server.py +384 -466
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/METADATA +36 -1
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/RECORD +9 -9
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/WHEEL +1 -1
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/entry_points.txt +0 -0
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/licenses/LICENSE +0 -0
- {mobile_mcp_ai-2.6.10.dist-info → mobile_mcp_ai-2.7.0.dist-info}/top_level.txt +0 -0
|
@@ -8,112 +8,28 @@
|
|
|
8
8
|
- 核心功能精简
|
|
9
9
|
- 保留 pytest 脚本生成
|
|
10
10
|
- 支持操作历史记录
|
|
11
|
+
- Token 优化模式(省钱)
|
|
11
12
|
"""
|
|
12
13
|
|
|
13
14
|
import asyncio
|
|
14
15
|
import time
|
|
15
16
|
import re
|
|
16
17
|
from pathlib import Path
|
|
17
|
-
from typing import Dict, List, Optional
|
|
18
|
+
from typing import Dict, List, Optional
|
|
18
19
|
from datetime import datetime
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
'Accept', 'ACCEPT', 'accept', 'Agree', 'AGREE', 'agree',
|
|
33
|
-
'Allow', 'ALLOW', 'allow', 'Confirm', 'CONFIRM', 'confirm',
|
|
34
|
-
'Got it', 'GOT IT', 'Done', 'DONE', 'Continue', 'CONTINUE',
|
|
35
|
-
]
|
|
36
|
-
|
|
37
|
-
# 关闭/取消按钮文本(次优先级 - 当没有确认按钮时使用)
|
|
38
|
-
CLOSE_BUTTON_TEXTS = [
|
|
39
|
-
# 符号类
|
|
40
|
-
'×', 'X', 'x', '✕', '✖', '╳',
|
|
41
|
-
# 中文关闭/拒绝类
|
|
42
|
-
'关闭', '取消', '跳过', '放弃', '算了', '忽略', '不了', '拒绝', '否',
|
|
43
|
-
'稍后再说', '下次再说', '以后再说', '暂不需要', '不再提示',
|
|
44
|
-
'不要', '残忍拒绝', '狠心离开', '我再想想', '暂不',
|
|
45
|
-
'不感兴趣', '跳过广告', '稍后更新',
|
|
46
|
-
'不同意', '仍不同意', '不同意并退出', # 这些优先级最低
|
|
47
|
-
# 英文关闭/拒绝类
|
|
48
|
-
'close', 'Close', 'CLOSE', 'Skip', 'skip', 'SKIP',
|
|
49
|
-
'Dismiss', 'dismiss', 'Cancel', 'cancel', 'CANCEL',
|
|
50
|
-
'Not now', 'NOT NOW', 'Later', 'later', 'LATER',
|
|
51
|
-
'No thanks', 'NO THANKS', 'No', 'NO', 'no',
|
|
52
|
-
'Ignore', 'IGNORE', 'Deny', 'DENY', 'Reject', 'REJECT',
|
|
53
|
-
]
|
|
54
|
-
|
|
55
|
-
# 关闭按钮描述关键词(模糊匹配 content-desc / resource-id)
|
|
56
|
-
# 确认类关键词(优先)
|
|
57
|
-
CONFIRM_BUTTON_KEYWORDS = [
|
|
58
|
-
'confirm', 'accept', 'agree', 'allow', 'ok', 'yes', 'done',
|
|
59
|
-
'确认', '同意', '允许', '确定',
|
|
60
|
-
]
|
|
61
|
-
|
|
62
|
-
# 关闭类关键词(次优先)
|
|
63
|
-
CLOSE_BUTTON_KEYWORDS = [
|
|
64
|
-
'close', 'dismiss', 'cancel', 'skip', 'ignore', 'deny', 'reject',
|
|
65
|
-
'关闭', '取消', '跳过', '忽略', '拒绝',
|
|
66
|
-
]
|
|
67
|
-
|
|
68
|
-
# 弹窗检测 - class 名称关键词(强特征)
|
|
69
|
-
POPUP_CLASS_KEYWORDS = [
|
|
70
|
-
'Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet',
|
|
71
|
-
'PopupWindow', 'DialogFragment', 'AlertDialog',
|
|
72
|
-
]
|
|
73
|
-
|
|
74
|
-
# 弹窗检测 - resource-id 关键词(强特征)
|
|
75
|
-
POPUP_ID_KEYWORDS = [
|
|
76
|
-
'dialog', 'popup', 'alert', 'modal', 'bottom_sheet',
|
|
77
|
-
'overlay', 'mask', 'dim', 'scrim',
|
|
78
|
-
]
|
|
79
|
-
|
|
80
|
-
# 广告弹窗 resource-id 关键词
|
|
81
|
-
AD_POPUP_KEYWORDS = [
|
|
82
|
-
'ad_close', 'ad_button', 'full_screen', 'interstitial',
|
|
83
|
-
'reward', 'close_icon', 'close_btn', 'ad_container',
|
|
84
|
-
'splash', 'banner', 'native_ad',
|
|
85
|
-
]
|
|
86
|
-
|
|
87
|
-
# 小元素排除关键词(避免误点)
|
|
88
|
-
EXCLUDE_ELEMENT_KEYWORDS = [
|
|
89
|
-
# 功能按钮
|
|
90
|
-
'more', 'menu', 'setting', 'settings', 'option', 'options',
|
|
91
|
-
'share', 'favorite', 'like', 'comment', 'follow', 'download',
|
|
92
|
-
'search', 'back', 'home', 'profile', 'notification', 'message',
|
|
93
|
-
# 播放控制
|
|
94
|
-
'play', 'pause', 'next', 'previous', 'volume', 'fullscreen',
|
|
95
|
-
'mute', 'unmute', 'forward', 'rewind', 'seek',
|
|
96
|
-
# 导航
|
|
97
|
-
'tab', 'navigation', 'drawer', 'sidebar',
|
|
98
|
-
# 编辑
|
|
99
|
-
'edit', 'delete', 'copy', 'paste', 'cut', 'select',
|
|
100
|
-
]
|
|
101
|
-
|
|
102
|
-
# 页面内容特征关键词(非弹窗)
|
|
103
|
-
PAGE_CONTENT_KEYWORDS = [
|
|
104
|
-
'video', 'player', 'recycler', 'list', 'scroll',
|
|
105
|
-
'viewpager', 'fragment', 'webview', 'content',
|
|
106
|
-
]
|
|
107
|
-
|
|
108
|
-
# 弹窗检测默认置信度阈值
|
|
109
|
-
DEFAULT_POPUP_CONFIDENCE_THRESHOLD = 0.6
|
|
110
|
-
|
|
111
|
-
# iOS 弹窗元素类型
|
|
112
|
-
IOS_POPUP_TYPES = [
|
|
113
|
-
'XCUIElementTypeAlert',
|
|
114
|
-
'XCUIElementTypeSheet',
|
|
115
|
-
'XCUIElementTypePopover',
|
|
116
|
-
]
|
|
21
|
+
# Token 优化配置(只精简格式,不限制数量,确保准确度)
|
|
22
|
+
try:
|
|
23
|
+
from mobile_mcp.config import Config
|
|
24
|
+
TOKEN_OPTIMIZATION = Config.TOKEN_OPTIMIZATION_ENABLED
|
|
25
|
+
MAX_ELEMENTS = Config.MAX_ELEMENTS_RETURN
|
|
26
|
+
MAX_SOM_ELEMENTS = Config.MAX_SOM_ELEMENTS_RETURN
|
|
27
|
+
COMPACT_RESPONSE = Config.COMPACT_RESPONSE
|
|
28
|
+
except ImportError:
|
|
29
|
+
TOKEN_OPTIMIZATION = True
|
|
30
|
+
MAX_ELEMENTS = 0 # 0 = 不限制
|
|
31
|
+
MAX_SOM_ELEMENTS = 0 # 0 = 不限制
|
|
32
|
+
COMPACT_RESPONSE = True
|
|
117
33
|
|
|
118
34
|
|
|
119
35
|
class BasicMobileToolsLite:
|
|
@@ -433,7 +349,7 @@ class BasicMobileToolsLite:
|
|
|
433
349
|
size = ios_client.wda.window_size()
|
|
434
350
|
screen_width, screen_height = size[0], size[1]
|
|
435
351
|
else:
|
|
436
|
-
return {"success": False, "
|
|
352
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
437
353
|
else:
|
|
438
354
|
self.client.u2.screenshot(str(temp_path))
|
|
439
355
|
info = self.client.u2.info
|
|
@@ -484,22 +400,14 @@ class BasicMobileToolsLite:
|
|
|
484
400
|
|
|
485
401
|
cropped_size = final_path.stat().st_size
|
|
486
402
|
|
|
403
|
+
# 返回结果
|
|
487
404
|
return {
|
|
488
405
|
"success": True,
|
|
489
406
|
"screenshot_path": str(final_path),
|
|
490
|
-
"screen_width": screen_width,
|
|
491
|
-
"screen_height": screen_height,
|
|
492
407
|
"image_width": img.width,
|
|
493
408
|
"image_height": img.height,
|
|
494
409
|
"crop_offset_x": crop_offset_x,
|
|
495
|
-
"crop_offset_y": crop_offset_y
|
|
496
|
-
"file_size": f"{cropped_size/1024:.1f}KB",
|
|
497
|
-
"message": f"🔍 局部截图已保存: {final_path}\n"
|
|
498
|
-
f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
|
|
499
|
-
f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
|
|
500
|
-
f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
|
|
501
|
-
f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
|
|
502
|
-
f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
|
|
410
|
+
"crop_offset_y": crop_offset_y
|
|
503
411
|
}
|
|
504
412
|
|
|
505
413
|
# ========== 情况2:全屏压缩截图 ==========
|
|
@@ -552,24 +460,14 @@ class BasicMobileToolsLite:
|
|
|
552
460
|
compressed_size = final_path.stat().st_size
|
|
553
461
|
saved_percent = (1 - compressed_size / original_size) * 100
|
|
554
462
|
|
|
463
|
+
# 返回结果
|
|
555
464
|
return {
|
|
556
465
|
"success": True,
|
|
557
466
|
"screenshot_path": str(final_path),
|
|
558
|
-
"
|
|
559
|
-
"
|
|
560
|
-
"original_img_width": original_img_width,
|
|
561
|
-
"original_img_height": original_img_height
|
|
562
|
-
"image_width": image_width, # 压缩后宽度(AI 看到的)
|
|
563
|
-
"image_height": image_height, # 压缩后高度(AI 看到的)
|
|
564
|
-
"original_size": f"{original_size/1024:.1f}KB",
|
|
565
|
-
"compressed_size": f"{compressed_size/1024:.1f}KB",
|
|
566
|
-
"saved_percent": f"{saved_percent:.0f}%",
|
|
567
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
568
|
-
f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
|
|
569
|
-
f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
|
|
570
|
-
f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
|
|
571
|
-
f" image_width={image_width}, image_height={image_height},\n"
|
|
572
|
-
f" original_img_width={original_img_width}, original_img_height={original_img_height}"
|
|
467
|
+
"image_width": image_width,
|
|
468
|
+
"image_height": image_height,
|
|
469
|
+
"original_img_width": original_img_width,
|
|
470
|
+
"original_img_height": original_img_height
|
|
573
471
|
}
|
|
574
472
|
|
|
575
473
|
# ========== 情况3:全屏不压缩截图 ==========
|
|
@@ -583,21 +481,12 @@ class BasicMobileToolsLite:
|
|
|
583
481
|
final_path = self.screenshot_dir / filename
|
|
584
482
|
temp_path.rename(final_path)
|
|
585
483
|
|
|
586
|
-
#
|
|
484
|
+
# 返回结果(不压缩时尺寸相同)
|
|
587
485
|
return {
|
|
588
486
|
"success": True,
|
|
589
487
|
"screenshot_path": str(final_path),
|
|
590
|
-
"
|
|
591
|
-
"
|
|
592
|
-
"original_img_width": img.width, # 截图实际尺寸
|
|
593
|
-
"original_img_height": img.height,
|
|
594
|
-
"image_width": img.width, # 未压缩,和原图一样
|
|
595
|
-
"image_height": img.height,
|
|
596
|
-
"file_size": f"{original_size/1024:.1f}KB",
|
|
597
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
598
|
-
f"📐 截图尺寸: {img.width}x{img.height}\n"
|
|
599
|
-
f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
|
|
600
|
-
f"💡 未压缩,坐标可直接使用"
|
|
488
|
+
"image_width": img.width,
|
|
489
|
+
"image_height": img.height
|
|
601
490
|
}
|
|
602
491
|
except ImportError:
|
|
603
492
|
# 如果没有 PIL,回退到原始方式(不压缩)
|
|
@@ -609,15 +498,11 @@ class BasicMobileToolsLite:
|
|
|
609
498
|
"""截图并添加网格坐标标注(用于精确定位元素)
|
|
610
499
|
|
|
611
500
|
在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
|
|
612
|
-
|
|
613
|
-
🎯 弹窗检测场景:
|
|
614
|
-
- show_popup_hints=True: 明确弹窗场景时使用(如调用 mobile_close_popup 前)
|
|
615
|
-
- show_popup_hints=False: 普通截图,不检测弹窗
|
|
501
|
+
如果检测到弹窗,会标注弹窗区域和可能的关闭按钮位置。
|
|
616
502
|
|
|
617
503
|
Args:
|
|
618
504
|
grid_size: 网格间距(像素),默认 100。建议值:50-200
|
|
619
|
-
show_popup_hints: 是否显示弹窗关闭按钮提示位置,默认
|
|
620
|
-
仅在明确弹窗场景时设置为 True,会标注弹窗区域和可能的关闭按钮位置。
|
|
505
|
+
show_popup_hints: 是否显示弹窗关闭按钮提示位置,默认 True
|
|
621
506
|
|
|
622
507
|
Returns:
|
|
623
508
|
包含标注截图路径和弹窗信息的字典
|
|
@@ -641,7 +526,7 @@ class BasicMobileToolsLite:
|
|
|
641
526
|
size = ios_client.wda.window_size()
|
|
642
527
|
screen_width, screen_height = size[0], size[1]
|
|
643
528
|
else:
|
|
644
|
-
return {"success": False, "
|
|
529
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
645
530
|
else:
|
|
646
531
|
self.client.u2.screenshot(str(temp_path))
|
|
647
532
|
info = self.client.u2.info
|
|
@@ -755,26 +640,16 @@ class BasicMobileToolsLite:
|
|
|
755
640
|
result = {
|
|
756
641
|
"success": True,
|
|
757
642
|
"screenshot_path": str(final_path),
|
|
758
|
-
"screen_width": screen_width,
|
|
759
|
-
"screen_height": screen_height,
|
|
760
643
|
"image_width": img_width,
|
|
761
644
|
"image_height": img_height,
|
|
762
|
-
"grid_size": grid_size
|
|
763
|
-
"message": f"📸 网格截图已保存: {final_path}\n"
|
|
764
|
-
f"📐 尺寸: {img_width}x{img_height}\n"
|
|
765
|
-
f"📏 网格间距: {grid_size}px"
|
|
645
|
+
"grid_size": grid_size
|
|
766
646
|
}
|
|
767
647
|
|
|
768
648
|
if popup_info:
|
|
769
|
-
result["
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
|
|
774
|
-
for pos in close_positions:
|
|
775
|
-
result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
|
|
776
|
-
else:
|
|
777
|
-
result["popup_detected"] = False
|
|
649
|
+
result["popup"] = popup_info["bounds"]
|
|
650
|
+
# 只返回前3个最可能的关闭按钮位置
|
|
651
|
+
if close_positions:
|
|
652
|
+
result["close_hints"] = [(p['x'], p['y']) for p in close_positions[:3]]
|
|
778
653
|
|
|
779
654
|
return result
|
|
780
655
|
|
|
@@ -783,17 +658,12 @@ class BasicMobileToolsLite:
|
|
|
783
658
|
except Exception as e:
|
|
784
659
|
return {"success": False, "message": f"❌ 网格截图失败: {e}"}
|
|
785
660
|
|
|
786
|
-
def take_screenshot_with_som(self
|
|
661
|
+
def take_screenshot_with_som(self) -> Dict:
|
|
787
662
|
"""Set-of-Mark 截图:给每个可点击元素标上数字(超级好用!)
|
|
788
663
|
|
|
789
664
|
在截图上给每个可点击元素画框并标上数字编号。
|
|
790
665
|
AI 看图后直接说"点击 3 号",然后调用 click_by_som(3) 即可。
|
|
791
666
|
|
|
792
|
-
Args:
|
|
793
|
-
check_popup: 是否检测弹窗,默认 False
|
|
794
|
-
- True: 明确弹窗场景时使用(如调用 mobile_close_popup 前)
|
|
795
|
-
- False: 普通截图,不检测弹窗
|
|
796
|
-
|
|
797
667
|
Returns:
|
|
798
668
|
包含标注截图和元素列表的字典
|
|
799
669
|
"""
|
|
@@ -816,7 +686,7 @@ class BasicMobileToolsLite:
|
|
|
816
686
|
size = ios_client.wda.window_size()
|
|
817
687
|
screen_width, screen_height = size[0], size[1]
|
|
818
688
|
else:
|
|
819
|
-
return {"success": False, "
|
|
689
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
820
690
|
else:
|
|
821
691
|
self.client.u2.screenshot(str(temp_path))
|
|
822
692
|
info = self.client.u2.info
|
|
@@ -930,12 +800,11 @@ class BasicMobileToolsLite:
|
|
|
930
800
|
'resource_id': elem.get('resource_id', '')
|
|
931
801
|
})
|
|
932
802
|
|
|
933
|
-
# 第3.5
|
|
803
|
+
# 第3.5步:检测弹窗区域(使用严格的置信度检测,避免误识别普通页面)
|
|
934
804
|
popup_bounds = None
|
|
935
805
|
popup_confidence = 0
|
|
936
806
|
|
|
937
|
-
|
|
938
|
-
if check_popup and not self._is_ios():
|
|
807
|
+
if not self._is_ios():
|
|
939
808
|
try:
|
|
940
809
|
# 使用严格的弹窗检测(置信度 >= 0.6 才认为是弹窗)
|
|
941
810
|
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
@@ -977,154 +846,22 @@ class BasicMobileToolsLite:
|
|
|
977
846
|
img.save(str(final_path), "JPEG", quality=85)
|
|
978
847
|
temp_path.unlink()
|
|
979
848
|
|
|
980
|
-
#
|
|
981
|
-
|
|
982
|
-
f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
|
|
983
|
-
for e in som_elements[:15] # 只显示前15个
|
|
984
|
-
])
|
|
985
|
-
if len(som_elements) > 15:
|
|
986
|
-
elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
|
|
987
|
-
|
|
988
|
-
# 构建弹窗提示文字
|
|
989
|
-
hints_text = ""
|
|
990
|
-
if popup_bounds:
|
|
991
|
-
hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
|
|
992
|
-
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
993
|
-
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
994
|
-
|
|
995
|
-
# 第5步:AI 分析图片内容(使用 Cursor AI 视觉能力)
|
|
996
|
-
ai_analysis = None
|
|
997
|
-
try:
|
|
998
|
-
ai_analysis = self._analyze_screenshot_with_ai(
|
|
999
|
-
str(final_path), screen_width, screen_height, img_width, img_height
|
|
1000
|
-
)
|
|
1001
|
-
except Exception as e:
|
|
1002
|
-
# AI 分析失败不影响主功能
|
|
1003
|
-
pass
|
|
1004
|
-
|
|
1005
|
-
# 构建返回消息
|
|
1006
|
-
message = f"📸 SoM 截图已保存: {final_path}\n"
|
|
1007
|
-
message += f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
1008
|
-
message += f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
1009
|
-
|
|
1010
|
-
# 添加 AI 分析提示(实际分析由 Cursor AI 在看到图片后自动完成)
|
|
1011
|
-
if ai_analysis:
|
|
1012
|
-
screenshot_path_for_ai = ai_analysis.get('screenshot_path', str(final_path))
|
|
1013
|
-
message += f"\n🤖 【AI 视觉分析请求】\n"
|
|
1014
|
-
message += f"📸 截图路径: {screenshot_path_for_ai}\n"
|
|
1015
|
-
message += f"📐 屏幕尺寸: {screen_width}x{screen_height},图片尺寸: {img_width}x{img_height}\n\n"
|
|
1016
|
-
message += f"请分析这张截图:\n"
|
|
1017
|
-
message += f"1. 识别页面主要内容\n"
|
|
1018
|
-
message += f"2. 检测是否有弹窗(对话框、提示框、广告、教程弹窗等)\n"
|
|
1019
|
-
message += f"3. 如果有弹窗,识别关闭按钮位置(×、关闭、知道了、好的、同意等)\n"
|
|
1020
|
-
message += f"4. 返回关闭按钮的精确位置:\n"
|
|
1021
|
-
message += f" - 像素坐标 (x, y)\n"
|
|
1022
|
-
message += f" - 百分比坐标 (x_percent, y_percent)\n"
|
|
1023
|
-
message += f" - 按钮文本(如'知道了'、'好的'等)\n\n"
|
|
1024
|
-
message += f"💡 分析结果格式(JSON):\n"
|
|
1025
|
-
message += f"{{\n"
|
|
1026
|
-
message += f" \"has_popup\": true/false,\n"
|
|
1027
|
-
message += f" \"popup_description\": \"弹窗描述\",\n"
|
|
1028
|
-
message += f" \"close_button\": {{\n"
|
|
1029
|
-
message += f" \"x\": 像素X坐标,\n"
|
|
1030
|
-
message += f" \"y\": 像素Y坐标,\n"
|
|
1031
|
-
message += f" \"x_percent\": X百分比(0-100),\n"
|
|
1032
|
-
message += f" \"y_percent\": Y百分比(0-100),\n"
|
|
1033
|
-
message += f" \"text\": \"按钮文本\"\n"
|
|
1034
|
-
message += f" }}\n"
|
|
1035
|
-
message += f"}}\n"
|
|
1036
|
-
|
|
1037
|
-
message += f"💡 使用方法:\n"
|
|
1038
|
-
message += f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
1039
|
-
message += f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
1040
|
-
|
|
1041
|
-
result = {
|
|
849
|
+
# 返回结果(Token 优化:不返回 elements 列表,已存储在 self._som_elements)
|
|
850
|
+
return {
|
|
1042
851
|
"success": True,
|
|
1043
852
|
"screenshot_path": str(final_path),
|
|
1044
853
|
"screen_width": screen_width,
|
|
1045
854
|
"screen_height": screen_height,
|
|
1046
|
-
"image_width": img_width,
|
|
1047
|
-
"image_height": img_height,
|
|
1048
855
|
"element_count": len(som_elements),
|
|
1049
|
-
"elements": som_elements,
|
|
1050
856
|
"popup_detected": popup_bounds is not None,
|
|
1051
|
-
"
|
|
1052
|
-
"message": message
|
|
857
|
+
"hint": "查看截图上的编号,用 click_by_som(编号) 点击"
|
|
1053
858
|
}
|
|
1054
859
|
|
|
1055
|
-
# 添加 AI 分析结果到返回字典
|
|
1056
|
-
if ai_analysis:
|
|
1057
|
-
result["ai_analysis"] = ai_analysis
|
|
1058
|
-
|
|
1059
|
-
return result
|
|
1060
|
-
|
|
1061
860
|
except ImportError:
|
|
1062
861
|
return {"success": False, "message": "❌ 需要安装 Pillow: pip install Pillow"}
|
|
1063
862
|
except Exception as e:
|
|
1064
863
|
return {"success": False, "message": f"❌ SoM 截图失败: {e}"}
|
|
1065
864
|
|
|
1066
|
-
def _analyze_screenshot_with_ai(self, screenshot_path: str, screen_width: int, screen_height: int,
|
|
1067
|
-
image_width: int, image_height: int) -> Optional[Dict]:
|
|
1068
|
-
"""准备 AI 分析截图内容
|
|
1069
|
-
|
|
1070
|
-
在 MCP 环境中,返回图片路径和分析提示,让 Cursor AI 自动分析图片。
|
|
1071
|
-
实际的 AI 分析由 Cursor AI 在看到图片后自动完成。
|
|
1072
|
-
|
|
1073
|
-
Args:
|
|
1074
|
-
screenshot_path: 截图文件路径
|
|
1075
|
-
screen_width: 屏幕宽度
|
|
1076
|
-
screen_height: 屏幕高度
|
|
1077
|
-
image_width: 图片宽度
|
|
1078
|
-
image_height: 图片高度
|
|
1079
|
-
|
|
1080
|
-
Returns:
|
|
1081
|
-
AI 分析准备信息字典
|
|
1082
|
-
"""
|
|
1083
|
-
try:
|
|
1084
|
-
from pathlib import Path
|
|
1085
|
-
|
|
1086
|
-
# 检查图片文件是否存在
|
|
1087
|
-
img_path = Path(screenshot_path)
|
|
1088
|
-
if not img_path.exists():
|
|
1089
|
-
return None
|
|
1090
|
-
|
|
1091
|
-
# 返回分析准备信息
|
|
1092
|
-
# 注意:实际的 AI 分析由 Cursor AI 在看到图片后自动完成
|
|
1093
|
-
return {
|
|
1094
|
-
"needs_ai_analysis": True,
|
|
1095
|
-
"screenshot_path": screenshot_path,
|
|
1096
|
-
"screen_size": {"width": screen_width, "height": screen_height},
|
|
1097
|
-
"image_size": {"width": image_width, "height": image_height},
|
|
1098
|
-
"analysis_instructions": (
|
|
1099
|
-
"🤖 【AI 视觉分析请求】\n"
|
|
1100
|
-
"请分析截图文件: {}\n\n"
|
|
1101
|
-
"分析要求:\n"
|
|
1102
|
-
"1. 识别页面主要内容\n"
|
|
1103
|
-
"2. 检测是否有弹窗(对话框、提示框、广告、教程弹窗等)\n"
|
|
1104
|
-
"3. 如果有弹窗,识别关闭按钮位置(×、关闭、知道了、好的、同意等)\n"
|
|
1105
|
-
"4. 返回关闭按钮的精确位置:\n"
|
|
1106
|
-
" - 像素坐标 (x, y)\n"
|
|
1107
|
-
" - 百分比坐标 (x_percent, y_percent)\n"
|
|
1108
|
-
" - 按钮文本(如'知道了'、'好的'等)\n\n"
|
|
1109
|
-
"屏幕尺寸: {}x{},图片尺寸: {}x{}\n\n"
|
|
1110
|
-
"请以 JSON 格式返回分析结果:\n"
|
|
1111
|
-
"{{\n"
|
|
1112
|
-
" \"has_popup\": true/false,\n"
|
|
1113
|
-
" \"popup_description\": \"弹窗描述\",\n"
|
|
1114
|
-
" \"close_button\": {{\n"
|
|
1115
|
-
" \"x\": 像素X坐标,\n"
|
|
1116
|
-
" \"y\": 像素Y坐标,\n"
|
|
1117
|
-
" \"x_percent\": X百分比(0-100),\n"
|
|
1118
|
-
" \"y_percent\": Y百分比(0-100),\n"
|
|
1119
|
-
" \"text\": \"按钮文本\"\n"
|
|
1120
|
-
" }}\n"
|
|
1121
|
-
"}}"
|
|
1122
|
-
).format(screenshot_path, screen_width, screen_height, image_width, image_height)
|
|
1123
|
-
}
|
|
1124
|
-
except Exception as e:
|
|
1125
|
-
# AI 分析准备失败不影响主功能
|
|
1126
|
-
return None
|
|
1127
|
-
|
|
1128
865
|
def click_by_som(self, index: int) -> Dict:
|
|
1129
866
|
"""根据 SoM 编号点击元素
|
|
1130
867
|
|
|
@@ -1198,7 +935,6 @@ class BasicMobileToolsLite:
|
|
|
1198
935
|
|
|
1199
936
|
return {
|
|
1200
937
|
"success": True,
|
|
1201
|
-
"message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
|
|
1202
938
|
"clicked": {
|
|
1203
939
|
"index": index,
|
|
1204
940
|
"desc": target['desc'],
|
|
@@ -1208,23 +944,7 @@ class BasicMobileToolsLite:
|
|
|
1208
944
|
}
|
|
1209
945
|
|
|
1210
946
|
except Exception as e:
|
|
1211
|
-
|
|
1212
|
-
popup_detected = False
|
|
1213
|
-
popup_hint = ""
|
|
1214
|
-
if not self._is_ios():
|
|
1215
|
-
try:
|
|
1216
|
-
page_analysis = self._analyze_page_for_popup()
|
|
1217
|
-
if page_analysis.get("has_popup"):
|
|
1218
|
-
popup_detected = True
|
|
1219
|
-
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1220
|
-
except Exception:
|
|
1221
|
-
pass
|
|
1222
|
-
|
|
1223
|
-
error_msg = f"❌ 点击失败: {e}\n💡 如果页面已变化,请重新调用 mobile_screenshot_with_som 刷新元素列表"
|
|
1224
|
-
if popup_detected:
|
|
1225
|
-
error_msg += popup_hint
|
|
1226
|
-
|
|
1227
|
-
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
947
|
+
return {"success": False, "message": f"❌ 点击失败: {e}\n💡 如果页面已变化,请重新调用 mobile_screenshot_with_som 刷新元素列表"}
|
|
1228
948
|
|
|
1229
949
|
def _take_screenshot_no_compress(self, description: str = "") -> Dict:
|
|
1230
950
|
"""截图(不压缩,PIL 不可用时的备用方案)"""
|
|
@@ -1248,7 +968,7 @@ class BasicMobileToolsLite:
|
|
|
1248
968
|
size = ios_client.wda.window_size()
|
|
1249
969
|
width, height = size[0], size[1]
|
|
1250
970
|
else:
|
|
1251
|
-
return {"success": False, "
|
|
971
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1252
972
|
else:
|
|
1253
973
|
self.client.u2.screenshot(str(screenshot_path))
|
|
1254
974
|
info = self.client.u2.info
|
|
@@ -1326,7 +1046,7 @@ class BasicMobileToolsLite:
|
|
|
1326
1046
|
size = ios_client.wda.window_size()
|
|
1327
1047
|
screen_width, screen_height = size[0], size[1]
|
|
1328
1048
|
else:
|
|
1329
|
-
return {"success": False, "
|
|
1049
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1330
1050
|
else:
|
|
1331
1051
|
info = self.client.u2.info
|
|
1332
1052
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1410,23 +1130,7 @@ class BasicMobileToolsLite:
|
|
|
1410
1130
|
"return_to_app": return_result
|
|
1411
1131
|
}
|
|
1412
1132
|
except Exception as e:
|
|
1413
|
-
|
|
1414
|
-
popup_detected = False
|
|
1415
|
-
popup_hint = ""
|
|
1416
|
-
if not self._is_ios():
|
|
1417
|
-
try:
|
|
1418
|
-
page_analysis = self._analyze_page_for_popup()
|
|
1419
|
-
if page_analysis.get("has_popup"):
|
|
1420
|
-
popup_detected = True
|
|
1421
|
-
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1422
|
-
except Exception:
|
|
1423
|
-
pass
|
|
1424
|
-
|
|
1425
|
-
error_msg = f"❌ 点击失败: {e}"
|
|
1426
|
-
if popup_detected:
|
|
1427
|
-
error_msg += popup_hint
|
|
1428
|
-
|
|
1429
|
-
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
1133
|
+
return {"success": False, "message": f"❌ 点击失败: {e}"}
|
|
1430
1134
|
|
|
1431
1135
|
def click_by_percent(self, x_percent: float, y_percent: float) -> Dict:
|
|
1432
1136
|
"""通过百分比坐标点击(跨设备兼容)
|
|
@@ -1457,14 +1161,14 @@ class BasicMobileToolsLite:
|
|
|
1457
1161
|
size = ios_client.wda.window_size()
|
|
1458
1162
|
width, height = size[0], size[1]
|
|
1459
1163
|
else:
|
|
1460
|
-
return {"success": False, "
|
|
1164
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1461
1165
|
else:
|
|
1462
1166
|
info = self.client.u2.info
|
|
1463
1167
|
width = info.get('displayWidth', 0)
|
|
1464
1168
|
height = info.get('displayHeight', 0)
|
|
1465
1169
|
|
|
1466
1170
|
if width == 0 or height == 0:
|
|
1467
|
-
return {"success": False, "
|
|
1171
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1468
1172
|
|
|
1469
1173
|
# 第2步:百分比转像素坐标
|
|
1470
1174
|
# 公式:像素 = 屏幕尺寸 × (百分比 / 100)
|
|
@@ -1485,15 +1189,13 @@ class BasicMobileToolsLite:
|
|
|
1485
1189
|
|
|
1486
1190
|
return {
|
|
1487
1191
|
"success": True,
|
|
1488
|
-
"message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
|
|
1489
|
-
"screen_size": {"width": width, "height": height},
|
|
1490
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1491
1192
|
"pixel": {"x": x, "y": y}
|
|
1492
1193
|
}
|
|
1493
1194
|
except Exception as e:
|
|
1494
1195
|
return {"success": False, "message": f"❌ 百分比点击失败: {e}"}
|
|
1495
1196
|
|
|
1496
|
-
def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None
|
|
1197
|
+
def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None,
|
|
1198
|
+
verify: Optional[str] = None) -> Dict:
|
|
1497
1199
|
"""通过文本点击 - 先查 XML 树,再精准匹配
|
|
1498
1200
|
|
|
1499
1201
|
Args:
|
|
@@ -1502,6 +1204,7 @@ class BasicMobileToolsLite:
|
|
|
1502
1204
|
position: 位置信息,当有多个相同文案时使用。支持:
|
|
1503
1205
|
- 垂直方向: "top"/"upper"/"上", "bottom"/"lower"/"下", "middle"/"center"/"中"
|
|
1504
1206
|
- 水平方向: "left"/"左", "right"/"右", "center"/"中"
|
|
1207
|
+
verify: 可选,点击后验证的文本。如果指定,会检查该文本是否出现在页面上
|
|
1505
1208
|
"""
|
|
1506
1209
|
try:
|
|
1507
1210
|
if self._is_ios():
|
|
@@ -1513,10 +1216,17 @@ class BasicMobileToolsLite:
|
|
|
1513
1216
|
if elem.exists:
|
|
1514
1217
|
elem.click()
|
|
1515
1218
|
time.sleep(0.3)
|
|
1516
|
-
# 使用标准记录格式
|
|
1517
1219
|
self._record_click('text', text, element_desc=text, locator_attr='text')
|
|
1518
|
-
|
|
1519
|
-
|
|
1220
|
+
# 验证逻辑
|
|
1221
|
+
if verify:
|
|
1222
|
+
return self._verify_after_click(verify, ios=True)
|
|
1223
|
+
# 返回页面文本摘要,方便确认页面变化
|
|
1224
|
+
page_texts = self._get_page_texts(10)
|
|
1225
|
+
return {"success": True, "page_texts": page_texts}
|
|
1226
|
+
# 控件树找不到,提示用视觉识别
|
|
1227
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1228
|
+
else:
|
|
1229
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1520
1230
|
else:
|
|
1521
1231
|
# 获取屏幕尺寸用于计算百分比
|
|
1522
1232
|
screen_width, screen_height = self.client.u2.window_size()
|
|
@@ -1537,17 +1247,20 @@ class BasicMobileToolsLite:
|
|
|
1537
1247
|
x_pct = round(cx / screen_width * 100, 1)
|
|
1538
1248
|
y_pct = round(cy / screen_height * 100, 1)
|
|
1539
1249
|
|
|
1540
|
-
#
|
|
1250
|
+
# 如果有位置参数,直接使用坐标点击
|
|
1541
1251
|
if position and bounds:
|
|
1542
1252
|
x = (bounds[0] + bounds[2]) // 2
|
|
1543
1253
|
y = (bounds[1] + bounds[3]) // 2
|
|
1544
1254
|
self.client.u2.click(x, y)
|
|
1545
1255
|
time.sleep(0.3)
|
|
1546
|
-
position_info = f" ({position})" if position else ""
|
|
1547
|
-
# 虽然用坐标点击,但记录时仍使用文本定位(脚本更稳定)
|
|
1548
1256
|
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1549
|
-
element_desc=f"{text}{
|
|
1550
|
-
|
|
1257
|
+
element_desc=f"{text}({position})", locator_attr=attr_type)
|
|
1258
|
+
# 验证逻辑
|
|
1259
|
+
if verify:
|
|
1260
|
+
return self._verify_after_click(verify)
|
|
1261
|
+
# 返回页面文本摘要
|
|
1262
|
+
page_texts = self._get_page_texts(10)
|
|
1263
|
+
return {"success": True, "page_texts": page_texts}
|
|
1551
1264
|
|
|
1552
1265
|
# 没有位置参数时,使用选择器定位
|
|
1553
1266
|
if attr_type == 'text':
|
|
@@ -1564,43 +1277,74 @@ class BasicMobileToolsLite:
|
|
|
1564
1277
|
if elem and elem.exists(timeout=1):
|
|
1565
1278
|
elem.click()
|
|
1566
1279
|
time.sleep(0.3)
|
|
1567
|
-
position_info = f" ({position})" if position else ""
|
|
1568
|
-
# 使用标准记录格式:文本定位
|
|
1569
1280
|
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1570
1281
|
element_desc=text, locator_attr=attr_type)
|
|
1571
|
-
|
|
1282
|
+
# 验证逻辑
|
|
1283
|
+
if verify:
|
|
1284
|
+
return self._verify_after_click(verify)
|
|
1285
|
+
# 返回页面文本摘要
|
|
1286
|
+
page_texts = self._get_page_texts(10)
|
|
1287
|
+
return {"success": True, "page_texts": page_texts}
|
|
1572
1288
|
|
|
1573
|
-
#
|
|
1289
|
+
# 选择器失败,用坐标兜底
|
|
1574
1290
|
if bounds:
|
|
1575
1291
|
x = (bounds[0] + bounds[2]) // 2
|
|
1576
1292
|
y = (bounds[1] + bounds[3]) // 2
|
|
1577
1293
|
self.client.u2.click(x, y)
|
|
1578
1294
|
time.sleep(0.3)
|
|
1579
|
-
position_info = f" ({position})" if position else ""
|
|
1580
|
-
# 选择器失败,用百分比作为兜底
|
|
1581
1295
|
self._record_click('percent', f"{x_pct}%,{y_pct}%", x_pct, y_pct,
|
|
1582
|
-
element_desc=
|
|
1583
|
-
|
|
1296
|
+
element_desc=text)
|
|
1297
|
+
# 验证逻辑
|
|
1298
|
+
if verify:
|
|
1299
|
+
return self._verify_after_click(verify)
|
|
1300
|
+
# 返回页面文本摘要
|
|
1301
|
+
page_texts = self._get_page_texts(10)
|
|
1302
|
+
return {"success": True, "page_texts": page_texts}
|
|
1584
1303
|
|
|
1585
|
-
|
|
1304
|
+
# 控件树找不到,提示用视觉识别
|
|
1305
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1586
1306
|
except Exception as e:
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1307
|
+
return {"success": False, "msg": str(e)}
|
|
1308
|
+
|
|
1309
|
+
def _verify_after_click(self, verify_text: str, ios: bool = False, timeout: float = 2.0) -> Dict:
|
|
1310
|
+
"""点击后验证期望文本是否出现
|
|
1311
|
+
|
|
1312
|
+
Args:
|
|
1313
|
+
verify_text: 期望出现的文本
|
|
1314
|
+
ios: 是否是 iOS 设备
|
|
1315
|
+
timeout: 验证超时时间
|
|
1316
|
+
|
|
1317
|
+
Returns:
|
|
1318
|
+
{"success": True, "verified": True/False, "hint": "..."}
|
|
1319
|
+
"""
|
|
1320
|
+
time.sleep(0.5) # 等待页面更新
|
|
1321
|
+
|
|
1322
|
+
try:
|
|
1323
|
+
if ios:
|
|
1324
|
+
ios_client = self._get_ios_client()
|
|
1325
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1326
|
+
exists = ios_client.wda(name=verify_text).exists or \
|
|
1327
|
+
ios_client.wda(label=verify_text).exists
|
|
1328
|
+
else:
|
|
1329
|
+
exists = False
|
|
1330
|
+
else:
|
|
1331
|
+
# Android: 检查文本或包含文本
|
|
1332
|
+
exists = self.client.u2(text=verify_text).exists(timeout=timeout) or \
|
|
1333
|
+
self.client.u2(textContains=verify_text).exists(timeout=0.5) or \
|
|
1334
|
+
self.client.u2(description=verify_text).exists(timeout=0.5)
|
|
1602
1335
|
|
|
1603
|
-
|
|
1336
|
+
if exists:
|
|
1337
|
+
return {"success": True, "verified": True}
|
|
1338
|
+
else:
|
|
1339
|
+
# 验证失败,提示可以截图确认
|
|
1340
|
+
return {
|
|
1341
|
+
"success": True, # 点击本身成功
|
|
1342
|
+
"verified": False,
|
|
1343
|
+
"expect": verify_text,
|
|
1344
|
+
"hint": "验证失败,可截图确认"
|
|
1345
|
+
}
|
|
1346
|
+
except Exception as e:
|
|
1347
|
+
return {"success": True, "verified": False, "hint": f"验证异常: {e}"}
|
|
1604
1348
|
|
|
1605
1349
|
def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
|
|
1606
1350
|
"""在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
|
|
@@ -1739,15 +1483,8 @@ class BasicMobileToolsLite:
|
|
|
1739
1483
|
return None
|
|
1740
1484
|
|
|
1741
1485
|
def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
|
|
1742
|
-
"""通过 resource-id
|
|
1743
|
-
|
|
1744
|
-
Args:
|
|
1745
|
-
resource_id: 元素的 resource-id
|
|
1746
|
-
index: 第几个元素(从 0 开始),默认 0 表示第一个
|
|
1747
|
-
"""
|
|
1486
|
+
"""通过 resource-id 点击"""
|
|
1748
1487
|
try:
|
|
1749
|
-
index_desc = f"[{index}]" if index > 0 else ""
|
|
1750
|
-
|
|
1751
1488
|
if self._is_ios():
|
|
1752
1489
|
ios_client = self._get_ios_client()
|
|
1753
1490
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
@@ -1755,49 +1492,31 @@ class BasicMobileToolsLite:
|
|
|
1755
1492
|
if not elem.exists:
|
|
1756
1493
|
elem = ios_client.wda(name=resource_id)
|
|
1757
1494
|
if elem.exists:
|
|
1758
|
-
# 获取所有匹配的元素
|
|
1759
1495
|
elements = elem.find_elements()
|
|
1760
1496
|
if index < len(elements):
|
|
1761
1497
|
elements[index].click()
|
|
1762
1498
|
time.sleep(0.3)
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}"}
|
|
1499
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1500
|
+
return {"success": True}
|
|
1766
1501
|
else:
|
|
1767
|
-
return {"success": False, "
|
|
1768
|
-
return {"success": False, "
|
|
1502
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{len(elements)}个)"}
|
|
1503
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1504
|
+
else:
|
|
1505
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1769
1506
|
else:
|
|
1770
1507
|
elem = self.client.u2(resourceId=resource_id)
|
|
1771
1508
|
if elem.exists(timeout=0.5):
|
|
1772
|
-
# 获取匹配元素数量
|
|
1773
1509
|
count = elem.count
|
|
1774
1510
|
if index < count:
|
|
1775
1511
|
elem[index].click()
|
|
1776
1512
|
time.sleep(0.3)
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}" + (f" (共 {count} 个)" if count > 1 else "")}
|
|
1513
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1514
|
+
return {"success": True}
|
|
1780
1515
|
else:
|
|
1781
|
-
return {"success": False, "
|
|
1782
|
-
return {"success": False, "
|
|
1516
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{count}个)"}
|
|
1517
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1783
1518
|
except Exception as e:
|
|
1784
|
-
|
|
1785
|
-
popup_detected = False
|
|
1786
|
-
popup_hint = ""
|
|
1787
|
-
if not self._is_ios():
|
|
1788
|
-
try:
|
|
1789
|
-
page_analysis = self._analyze_page_for_popup()
|
|
1790
|
-
if page_analysis.get("has_popup"):
|
|
1791
|
-
popup_detected = True
|
|
1792
|
-
popup_hint = "\n🎯 检测到弹窗(异常情况),可能影响操作,建议先调用 mobile_close_popup() 关闭弹窗"
|
|
1793
|
-
except Exception:
|
|
1794
|
-
pass
|
|
1795
|
-
|
|
1796
|
-
error_msg = f"❌ 点击失败: {e}"
|
|
1797
|
-
if popup_detected:
|
|
1798
|
-
error_msg += popup_hint
|
|
1799
|
-
|
|
1800
|
-
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
1519
|
+
return {"success": False, "msg": str(e)}
|
|
1801
1520
|
|
|
1802
1521
|
# ==================== 长按操作 ====================
|
|
1803
1522
|
|
|
@@ -1831,7 +1550,7 @@ class BasicMobileToolsLite:
|
|
|
1831
1550
|
size = ios_client.wda.window_size()
|
|
1832
1551
|
screen_width, screen_height = size[0], size[1]
|
|
1833
1552
|
else:
|
|
1834
|
-
return {"success": False, "
|
|
1553
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1835
1554
|
else:
|
|
1836
1555
|
info = self.client.u2.info
|
|
1837
1556
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1884,23 +1603,11 @@ class BasicMobileToolsLite:
|
|
|
1884
1603
|
|
|
1885
1604
|
if converted:
|
|
1886
1605
|
if conversion_type == "crop_offset":
|
|
1887
|
-
return {
|
|
1888
|
-
"success": True,
|
|
1889
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1890
|
-
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
1891
|
-
}
|
|
1606
|
+
return {"success": True}
|
|
1892
1607
|
else:
|
|
1893
|
-
return {
|
|
1894
|
-
"success": True,
|
|
1895
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1896
|
-
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
1897
|
-
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
1898
|
-
}
|
|
1608
|
+
return {"success": True}
|
|
1899
1609
|
else:
|
|
1900
|
-
return {
|
|
1901
|
-
"success": True,
|
|
1902
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
|
|
1903
|
-
}
|
|
1610
|
+
return {"success": True}
|
|
1904
1611
|
except Exception as e:
|
|
1905
1612
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1906
1613
|
|
|
@@ -1929,14 +1636,14 @@ class BasicMobileToolsLite:
|
|
|
1929
1636
|
size = ios_client.wda.window_size()
|
|
1930
1637
|
width, height = size[0], size[1]
|
|
1931
1638
|
else:
|
|
1932
|
-
return {"success": False, "
|
|
1639
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1933
1640
|
else:
|
|
1934
1641
|
info = self.client.u2.info
|
|
1935
1642
|
width = info.get('displayWidth', 0)
|
|
1936
1643
|
height = info.get('displayHeight', 0)
|
|
1937
1644
|
|
|
1938
1645
|
if width == 0 or height == 0:
|
|
1939
|
-
return {"success": False, "
|
|
1646
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1940
1647
|
|
|
1941
1648
|
# 第2步:百分比转像素坐标
|
|
1942
1649
|
x = int(width * x_percent / 100)
|
|
@@ -1958,13 +1665,7 @@ class BasicMobileToolsLite:
|
|
|
1958
1665
|
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1959
1666
|
x_percent, y_percent, element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1960
1667
|
|
|
1961
|
-
return {
|
|
1962
|
-
"success": True,
|
|
1963
|
-
"message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
|
|
1964
|
-
"screen_size": {"width": width, "height": height},
|
|
1965
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1966
|
-
"pixel": {"x": x, "y": y},
|
|
1967
|
-
"duration": duration
|
|
1668
|
+
return {"success": True
|
|
1968
1669
|
}
|
|
1969
1670
|
except Exception as e:
|
|
1970
1671
|
return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
|
|
@@ -1994,8 +1695,8 @@ class BasicMobileToolsLite:
|
|
|
1994
1695
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1995
1696
|
time.sleep(0.3)
|
|
1996
1697
|
self._record_long_press('text', text, duration, element_desc=text, locator_attr='text')
|
|
1997
|
-
return {"success": True
|
|
1998
|
-
return {"success": False, "
|
|
1698
|
+
return {"success": True}
|
|
1699
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1999
1700
|
else:
|
|
2000
1701
|
# 获取屏幕尺寸用于计算百分比
|
|
2001
1702
|
screen_width, screen_height = self.client.u2.window_size()
|
|
@@ -2033,7 +1734,7 @@ class BasicMobileToolsLite:
|
|
|
2033
1734
|
time.sleep(0.3)
|
|
2034
1735
|
self._record_long_press('text', attr_value, duration, x_pct, y_pct,
|
|
2035
1736
|
element_desc=text, locator_attr=attr_type)
|
|
2036
|
-
return {"success": True
|
|
1737
|
+
return {"success": True}
|
|
2037
1738
|
|
|
2038
1739
|
# 如果选择器失败,用坐标兜底
|
|
2039
1740
|
if bounds:
|
|
@@ -2043,9 +1744,9 @@ class BasicMobileToolsLite:
|
|
|
2043
1744
|
time.sleep(0.3)
|
|
2044
1745
|
self._record_long_press('percent', f"{x_pct}%,{y_pct}%", duration, x_pct, y_pct,
|
|
2045
1746
|
element_desc=text)
|
|
2046
|
-
return {"success": True
|
|
1747
|
+
return {"success": True}
|
|
2047
1748
|
|
|
2048
|
-
return {"success": False, "
|
|
1749
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
2049
1750
|
except Exception as e:
|
|
2050
1751
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
2051
1752
|
|
|
@@ -2073,8 +1774,8 @@ class BasicMobileToolsLite:
|
|
|
2073
1774
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
2074
1775
|
time.sleep(0.3)
|
|
2075
1776
|
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
2076
|
-
return {"success": True
|
|
2077
|
-
return {"success": False, "
|
|
1777
|
+
return {"success": True}
|
|
1778
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
2078
1779
|
else:
|
|
2079
1780
|
elem = self.client.u2(resourceId=resource_id)
|
|
2080
1781
|
if elem.exists(timeout=0.5):
|
|
@@ -2082,7 +1783,7 @@ class BasicMobileToolsLite:
|
|
|
2082
1783
|
time.sleep(0.3)
|
|
2083
1784
|
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
2084
1785
|
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
2085
|
-
return {"success": False, "
|
|
1786
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
2086
1787
|
except Exception as e:
|
|
2087
1788
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
2088
1789
|
|
|
@@ -2339,15 +2040,8 @@ class BasicMobileToolsLite:
|
|
|
2339
2040
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
2340
2041
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
2341
2042
|
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
x=x,
|
|
2345
|
-
y=y,
|
|
2346
|
-
x_percent=x_percent,
|
|
2347
|
-
y_percent=y_percent,
|
|
2348
|
-
ref=f"coords_{x}_{y}",
|
|
2349
|
-
text=text
|
|
2350
|
-
)
|
|
2043
|
+
# 使用标准记录格式
|
|
2044
|
+
self._record_input(text, 'percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent)
|
|
2351
2045
|
|
|
2352
2046
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2353
2047
|
app_check = self._check_app_switched()
|
|
@@ -2377,16 +2071,13 @@ class BasicMobileToolsLite:
|
|
|
2377
2071
|
|
|
2378
2072
|
# ==================== 导航操作 ====================
|
|
2379
2073
|
|
|
2380
|
-
async def swipe(self, direction: str, y: Optional[int] = None, y_percent: Optional[float] = None
|
|
2381
|
-
distance: Optional[int] = None, distance_percent: Optional[float] = None) -> Dict:
|
|
2074
|
+
async def swipe(self, direction: str, y: Optional[int] = None, y_percent: Optional[float] = None) -> Dict:
|
|
2382
2075
|
"""滑动屏幕
|
|
2383
2076
|
|
|
2384
2077
|
Args:
|
|
2385
2078
|
direction: 滑动方向 (up/down/left/right)
|
|
2386
2079
|
y: 左右滑动时指定的高度坐标(像素)
|
|
2387
2080
|
y_percent: 左右滑动时指定的高度百分比 (0-100)
|
|
2388
|
-
distance: 横向滑动时指定的滑动距离(像素),仅用于 left/right
|
|
2389
|
-
distance_percent: 横向滑动时指定的滑动距离百分比 (0-100),仅用于 left/right
|
|
2390
2081
|
"""
|
|
2391
2082
|
try:
|
|
2392
2083
|
if self._is_ios():
|
|
@@ -2395,7 +2086,7 @@ class BasicMobileToolsLite:
|
|
|
2395
2086
|
size = ios_client.wda.window_size()
|
|
2396
2087
|
width, height = size[0], size[1]
|
|
2397
2088
|
else:
|
|
2398
|
-
return {"success": False, "
|
|
2089
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
2399
2090
|
else:
|
|
2400
2091
|
width, height = self.client.u2.window_size()
|
|
2401
2092
|
|
|
@@ -2413,53 +2104,20 @@ class BasicMobileToolsLite:
|
|
|
2413
2104
|
swipe_y = y
|
|
2414
2105
|
else:
|
|
2415
2106
|
swipe_y = center_y
|
|
2416
|
-
|
|
2417
|
-
# 计算横向滑动距离
|
|
2418
|
-
if distance_percent is not None:
|
|
2419
|
-
if not (0 <= distance_percent <= 100):
|
|
2420
|
-
return {"success": False, "message": f"❌ distance_percent 必须在 0-100 之间: {distance_percent}"}
|
|
2421
|
-
swipe_distance = int(width * distance_percent / 100)
|
|
2422
|
-
elif distance is not None:
|
|
2423
|
-
if distance <= 0:
|
|
2424
|
-
return {"success": False, "message": f"❌ distance 必须大于 0: {distance}"}
|
|
2425
|
-
if distance > width:
|
|
2426
|
-
return {"success": False, "message": f"❌ distance 不能超过屏幕宽度 ({width}): {distance}"}
|
|
2427
|
-
swipe_distance = distance
|
|
2428
|
-
else:
|
|
2429
|
-
# 默认滑动距离:屏幕宽度的 60%(从 0.8 到 0.2)
|
|
2430
|
-
swipe_distance = int(width * 0.6)
|
|
2431
|
-
|
|
2432
|
-
# 计算起始和结束位置
|
|
2433
|
-
if direction == 'left':
|
|
2434
|
-
# 从右向左滑动:起始点在右侧,结束点在左侧
|
|
2435
|
-
# 确保起始点不超出屏幕右边界
|
|
2436
|
-
start_x = min(center_x + swipe_distance // 2, width - 10)
|
|
2437
|
-
end_x = start_x - swipe_distance
|
|
2438
|
-
# 确保结束点不超出屏幕左边界
|
|
2439
|
-
if end_x < 10:
|
|
2440
|
-
end_x = 10
|
|
2441
|
-
start_x = min(end_x + swipe_distance, width - 10)
|
|
2442
|
-
else: # right
|
|
2443
|
-
# 从左向右滑动:起始点在左侧,结束点在右侧
|
|
2444
|
-
# 确保起始点不超出屏幕左边界
|
|
2445
|
-
start_x = max(center_x - swipe_distance // 2, 10)
|
|
2446
|
-
end_x = start_x + swipe_distance
|
|
2447
|
-
# 确保结束点不超出屏幕右边界
|
|
2448
|
-
if end_x > width - 10:
|
|
2449
|
-
end_x = width - 10
|
|
2450
|
-
start_x = max(end_x - swipe_distance, 10)
|
|
2451
|
-
|
|
2452
|
-
x1, y1, x2, y2 = start_x, swipe_y, end_x, swipe_y
|
|
2453
2107
|
else:
|
|
2454
2108
|
swipe_y = center_y
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2109
|
+
|
|
2110
|
+
swipe_map = {
|
|
2111
|
+
'up': (center_x, int(height * 0.8), center_x, int(height * 0.2)),
|
|
2112
|
+
'down': (center_x, int(height * 0.2), center_x, int(height * 0.8)),
|
|
2113
|
+
'left': (int(width * 0.8), swipe_y, int(width * 0.2), swipe_y),
|
|
2114
|
+
'right': (int(width * 0.2), swipe_y, int(width * 0.8), swipe_y),
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
if direction not in swipe_map:
|
|
2118
|
+
return {"success": False, "message": f"❌ 不支持的方向: {direction}"}
|
|
2119
|
+
|
|
2120
|
+
x1, y1, x2, y2 = swipe_map[direction]
|
|
2463
2121
|
|
|
2464
2122
|
if self._is_ios():
|
|
2465
2123
|
ios_client.wda.swipe(x1, y1, x2, y2)
|
|
@@ -2480,21 +2138,10 @@ class BasicMobileToolsLite:
|
|
|
2480
2138
|
# 构建返回消息
|
|
2481
2139
|
msg = f"✅ 滑动成功: {direction}"
|
|
2482
2140
|
if direction in ['left', 'right']:
|
|
2483
|
-
msg_parts = []
|
|
2484
2141
|
if y_percent is not None:
|
|
2485
|
-
|
|
2142
|
+
msg += f" (高度: {y_percent}% = {swipe_y}px)"
|
|
2486
2143
|
elif y is not None:
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
if distance_percent is not None:
|
|
2490
|
-
msg_parts.append(f"距离: {distance_percent}% = {swipe_distance}px")
|
|
2491
|
-
elif distance is not None:
|
|
2492
|
-
msg_parts.append(f"距离: {distance}px")
|
|
2493
|
-
else:
|
|
2494
|
-
msg_parts.append(f"距离: 默认 {swipe_distance}px")
|
|
2495
|
-
|
|
2496
|
-
if msg_parts:
|
|
2497
|
-
msg += f" ({', '.join(msg_parts)})"
|
|
2144
|
+
msg += f" (高度: {y}px)"
|
|
2498
2145
|
|
|
2499
2146
|
# 如果检测到应用跳转,添加警告和返回结果
|
|
2500
2147
|
if app_check['switched']:
|
|
@@ -2535,194 +2182,34 @@ class BasicMobileToolsLite:
|
|
|
2535
2182
|
ios_client.wda.send_keys('\n')
|
|
2536
2183
|
elif ios_key == 'home':
|
|
2537
2184
|
ios_client.wda.home()
|
|
2538
|
-
return {"success": True
|
|
2539
|
-
return {"success": False, "
|
|
2185
|
+
return {"success": True}
|
|
2186
|
+
return {"success": False, "msg": f"iOS不支持{key}"}
|
|
2540
2187
|
else:
|
|
2541
2188
|
keycode = key_map.get(key.lower())
|
|
2542
2189
|
if keycode:
|
|
2543
2190
|
self.client.u2.shell(f'input keyevent {keycode}')
|
|
2544
2191
|
self._record_key(key)
|
|
2545
|
-
return {"success": True
|
|
2546
|
-
return {"success": False, "
|
|
2192
|
+
return {"success": True}
|
|
2193
|
+
return {"success": False, "msg": f"不支持按键{key}"}
|
|
2547
2194
|
except Exception as e:
|
|
2548
2195
|
return {"success": False, "message": f"❌ 按键失败: {e}"}
|
|
2549
2196
|
|
|
2550
2197
|
def wait(self, seconds: float) -> Dict:
|
|
2551
2198
|
"""等待指定时间"""
|
|
2552
2199
|
time.sleep(seconds)
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
- 如果进度条未显示,先点击播放区域显示控制栏,再拖动
|
|
2562
|
-
|
|
2563
|
-
Args:
|
|
2564
|
-
direction: 拖动方向,'left'(倒退)或 'right'(前进),默认 'right'
|
|
2565
|
-
distance_percent: 拖动距离百分比 (0-100),默认 30%
|
|
2566
|
-
y_percent: 进度条的垂直位置百分比 (0-100),如果未指定则自动检测
|
|
2567
|
-
y: 进度条的垂直位置坐标(像素),如果未指定则自动检测
|
|
2568
|
-
"""
|
|
2569
|
-
try:
|
|
2570
|
-
import xml.etree.ElementTree as ET
|
|
2571
|
-
import re
|
|
2572
|
-
|
|
2573
|
-
if self._is_ios():
|
|
2574
|
-
return {"success": False, "message": "❌ iOS 暂不支持,请使用 mobile_swipe"}
|
|
2575
|
-
|
|
2576
|
-
if direction not in ['left', 'right']:
|
|
2577
|
-
return {"success": False, "message": f"❌ 拖动方向必须是 'left' 或 'right': {direction}"}
|
|
2578
|
-
|
|
2579
|
-
screen_width, screen_height = self.client.u2.window_size()
|
|
2580
|
-
|
|
2581
|
-
# 获取 XML 查找进度条
|
|
2582
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2583
|
-
root = ET.fromstring(xml_string)
|
|
2584
|
-
|
|
2585
|
-
progress_bar_found = False
|
|
2586
|
-
progress_bar_y = None
|
|
2587
|
-
progress_bar_y_percent = None
|
|
2588
|
-
|
|
2589
|
-
# 查找进度条元素(SeekBar、ProgressBar)
|
|
2590
|
-
for elem in root.iter():
|
|
2591
|
-
class_name = elem.attrib.get('class', '')
|
|
2592
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2593
|
-
bounds_str = elem.attrib.get('bounds', '')
|
|
2594
|
-
|
|
2595
|
-
# 检查是否是进度条
|
|
2596
|
-
is_progress_bar = (
|
|
2597
|
-
'SeekBar' in class_name or
|
|
2598
|
-
'ProgressBar' in class_name or
|
|
2599
|
-
'progress' in resource_id.lower() or
|
|
2600
|
-
'seek' in resource_id.lower()
|
|
2601
|
-
)
|
|
2602
|
-
|
|
2603
|
-
if is_progress_bar and bounds_str:
|
|
2604
|
-
# 解析 bounds 获取进度条位置
|
|
2605
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2606
|
-
if match:
|
|
2607
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2608
|
-
center_y = (y1 + y2) // 2
|
|
2609
|
-
progress_bar_y = center_y
|
|
2610
|
-
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2611
|
-
progress_bar_found = True
|
|
2612
|
-
break
|
|
2613
|
-
|
|
2614
|
-
# 如果未找到进度条,尝试点击播放区域显示控制栏
|
|
2615
|
-
if not progress_bar_found:
|
|
2616
|
-
# 点击屏幕中心显示控制栏
|
|
2617
|
-
center_x, center_y = screen_width // 2, screen_height // 2
|
|
2618
|
-
self.client.u2.click(center_x, center_y)
|
|
2619
|
-
time.sleep(0.5)
|
|
2620
|
-
|
|
2621
|
-
# 再次查找进度条
|
|
2622
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2623
|
-
root = ET.fromstring(xml_string)
|
|
2624
|
-
|
|
2625
|
-
for elem in root.iter():
|
|
2626
|
-
class_name = elem.attrib.get('class', '')
|
|
2627
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2628
|
-
bounds_str = elem.attrib.get('bounds', '')
|
|
2629
|
-
|
|
2630
|
-
is_progress_bar = (
|
|
2631
|
-
'SeekBar' in class_name or
|
|
2632
|
-
'ProgressBar' in class_name or
|
|
2633
|
-
'progress' in resource_id.lower() or
|
|
2634
|
-
'seek' in resource_id.lower()
|
|
2635
|
-
)
|
|
2636
|
-
|
|
2637
|
-
if is_progress_bar and bounds_str:
|
|
2638
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2639
|
-
if match:
|
|
2640
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2641
|
-
center_y = (y1 + y2) // 2
|
|
2642
|
-
progress_bar_y = center_y
|
|
2643
|
-
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2644
|
-
progress_bar_found = True
|
|
2645
|
-
break
|
|
2646
|
-
|
|
2647
|
-
# 确定使用的高度位置
|
|
2648
|
-
if y_percent is not None:
|
|
2649
|
-
swipe_y = int(screen_height * y_percent / 100)
|
|
2650
|
-
used_y_percent = y_percent
|
|
2651
|
-
elif y is not None:
|
|
2652
|
-
swipe_y = y
|
|
2653
|
-
used_y_percent = round(y / screen_height * 100, 1)
|
|
2654
|
-
elif progress_bar_found:
|
|
2655
|
-
swipe_y = progress_bar_y
|
|
2656
|
-
used_y_percent = progress_bar_y_percent
|
|
2657
|
-
else:
|
|
2658
|
-
# 默认使用屏幕底部附近(进度条常见位置)
|
|
2659
|
-
swipe_y = int(screen_height * 0.91)
|
|
2660
|
-
used_y_percent = 91.0
|
|
2661
|
-
|
|
2662
|
-
# 计算滑动距离
|
|
2663
|
-
swipe_distance = int(screen_width * distance_percent / 100)
|
|
2664
|
-
|
|
2665
|
-
# 计算起始和结束位置
|
|
2666
|
-
center_x = screen_width // 2
|
|
2667
|
-
if direction == 'left':
|
|
2668
|
-
start_x = min(center_x + swipe_distance // 2, screen_width - 10)
|
|
2669
|
-
end_x = start_x - swipe_distance
|
|
2670
|
-
if end_x < 10:
|
|
2671
|
-
end_x = 10
|
|
2672
|
-
start_x = min(end_x + swipe_distance, screen_width - 10)
|
|
2673
|
-
else: # right
|
|
2674
|
-
start_x = max(center_x - swipe_distance // 2, 10)
|
|
2675
|
-
end_x = start_x + swipe_distance
|
|
2676
|
-
if end_x > screen_width - 10:
|
|
2677
|
-
end_x = screen_width - 10
|
|
2678
|
-
start_x = max(end_x - swipe_distance, 10)
|
|
2679
|
-
|
|
2680
|
-
# 执行拖动
|
|
2681
|
-
self.client.u2.swipe(start_x, swipe_y, end_x, swipe_y, duration=0.5)
|
|
2682
|
-
time.sleep(0.3)
|
|
2683
|
-
|
|
2684
|
-
# 记录操作
|
|
2685
|
-
self._record_swipe(direction)
|
|
2686
|
-
|
|
2687
|
-
# 检查应用是否跳转
|
|
2688
|
-
app_check = self._check_app_switched()
|
|
2689
|
-
return_result = None
|
|
2690
|
-
if app_check['switched']:
|
|
2691
|
-
return_result = self._return_to_target_app()
|
|
2692
|
-
|
|
2693
|
-
# 构建返回消息
|
|
2694
|
-
msg = f"✅ 进度条拖动成功: {direction} (高度: {used_y_percent}%, 距离: {distance_percent}%)"
|
|
2695
|
-
if not progress_bar_found:
|
|
2696
|
-
msg += "\n💡 已自动点击播放区域显示控制栏"
|
|
2697
|
-
else:
|
|
2698
|
-
msg += "\n💡 进度条已显示,直接拖动"
|
|
2699
|
-
|
|
2700
|
-
if app_check['switched']:
|
|
2701
|
-
msg += f"\n{app_check['message']}"
|
|
2702
|
-
if return_result and return_result.get('success'):
|
|
2703
|
-
msg += f"\n{return_result['message']}"
|
|
2704
|
-
|
|
2705
|
-
return {
|
|
2706
|
-
"success": True,
|
|
2707
|
-
"message": msg,
|
|
2708
|
-
"progress_bar_found": progress_bar_found,
|
|
2709
|
-
"y_percent": used_y_percent,
|
|
2710
|
-
"distance_percent": distance_percent,
|
|
2711
|
-
"direction": direction,
|
|
2712
|
-
"app_check": app_check,
|
|
2713
|
-
"return_to_app": return_result
|
|
2714
|
-
}
|
|
2715
|
-
|
|
2716
|
-
except Exception as e:
|
|
2717
|
-
return {"success": False, "message": f"❌ 拖动进度条失败: {e}"}
|
|
2200
|
+
# 记录等待操作
|
|
2201
|
+
record = {
|
|
2202
|
+
'action': 'wait',
|
|
2203
|
+
'timestamp': datetime.now().isoformat(),
|
|
2204
|
+
'seconds': seconds,
|
|
2205
|
+
}
|
|
2206
|
+
self.operation_history.append(record)
|
|
2207
|
+
return {"success": True}
|
|
2718
2208
|
|
|
2719
2209
|
# ==================== 应用管理 ====================
|
|
2720
2210
|
|
|
2721
2211
|
async def launch_app(self, package_name: str) -> Dict:
|
|
2722
|
-
"""启动应用
|
|
2723
|
-
|
|
2724
|
-
启动应用后会自动检测弹窗(启动应用场景)
|
|
2725
|
-
"""
|
|
2212
|
+
"""启动应用"""
|
|
2726
2213
|
try:
|
|
2727
2214
|
if self._is_ios():
|
|
2728
2215
|
ios_client = self._get_ios_client()
|
|
@@ -2746,45 +2233,9 @@ class BasicMobileToolsLite:
|
|
|
2746
2233
|
|
|
2747
2234
|
self._record_operation('launch_app', package_name=package_name)
|
|
2748
2235
|
|
|
2749
|
-
|
|
2750
|
-
popup_detected = False
|
|
2751
|
-
popup_message = ""
|
|
2752
|
-
if not self._is_ios():
|
|
2753
|
-
try:
|
|
2754
|
-
# 检测弹窗
|
|
2755
|
-
page_analysis = self._analyze_page_for_popup()
|
|
2756
|
-
if page_analysis.get("has_popup"):
|
|
2757
|
-
popup_detected = True
|
|
2758
|
-
popup_info = page_analysis.get("popup_info", {})
|
|
2759
|
-
popup_message = f"\n🎯 检测到弹窗(启动应用场景)\n💡 如需关闭弹窗,请调用 mobile_close_popup()"
|
|
2760
|
-
except Exception:
|
|
2761
|
-
pass # 弹窗检测失败不影响启动流程
|
|
2762
|
-
|
|
2763
|
-
message = f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载\n📱 已设置应用状态监测"
|
|
2764
|
-
if popup_detected:
|
|
2765
|
-
message += popup_message
|
|
2766
|
-
|
|
2767
|
-
return {
|
|
2768
|
-
"success": True,
|
|
2769
|
-
"message": message,
|
|
2770
|
-
"popup_detected": popup_detected
|
|
2771
|
-
}
|
|
2236
|
+
return {"success": True}
|
|
2772
2237
|
except Exception as e:
|
|
2773
|
-
|
|
2774
|
-
popup_detected = False
|
|
2775
|
-
if not self._is_ios():
|
|
2776
|
-
try:
|
|
2777
|
-
page_analysis = self._analyze_page_for_popup()
|
|
2778
|
-
if page_analysis.get("has_popup"):
|
|
2779
|
-
popup_detected = True
|
|
2780
|
-
except Exception:
|
|
2781
|
-
pass
|
|
2782
|
-
|
|
2783
|
-
error_msg = f"❌ 启动失败: {e}"
|
|
2784
|
-
if popup_detected:
|
|
2785
|
-
error_msg += "\n🎯 检测到弹窗(异常情况),可能影响启动,建议先关闭弹窗"
|
|
2786
|
-
|
|
2787
|
-
return {"success": False, "message": error_msg, "popup_detected": popup_detected}
|
|
2238
|
+
return {"success": False, "message": f"❌ 启动失败: {e}"}
|
|
2788
2239
|
|
|
2789
2240
|
def terminate_app(self, package_name: str) -> Dict:
|
|
2790
2241
|
"""终止应用"""
|
|
@@ -2795,9 +2246,9 @@ class BasicMobileToolsLite:
|
|
|
2795
2246
|
ios_client.wda.app_terminate(package_name)
|
|
2796
2247
|
else:
|
|
2797
2248
|
self.client.u2.app_stop(package_name)
|
|
2798
|
-
return {"success": True
|
|
2249
|
+
return {"success": True}
|
|
2799
2250
|
except Exception as e:
|
|
2800
|
-
return {"success": False, "
|
|
2251
|
+
return {"success": False, "msg": str(e)}
|
|
2801
2252
|
|
|
2802
2253
|
def list_apps(self, filter_keyword: str = "") -> Dict:
|
|
2803
2254
|
"""列出已安装应用"""
|
|
@@ -2909,6 +2360,35 @@ class BasicMobileToolsLite:
|
|
|
2909
2360
|
'_shadow', 'shadow_', '_divider', 'divider_', '_line', 'line_'
|
|
2910
2361
|
}
|
|
2911
2362
|
|
|
2363
|
+
# 状态栏相关关键词(这些元素对测试没有意义,直接过滤)
|
|
2364
|
+
STATUS_BAR_KEYWORDS = {
|
|
2365
|
+
'status_bar', 'statusbar', 'notification_icon', 'notificationicons',
|
|
2366
|
+
'system_icons', 'statusicons', 'battery', 'wifi_', 'wifi_combo',
|
|
2367
|
+
'wifi_group', 'wifi_signal', 'wifi_in', 'wifi_out', 'signal_',
|
|
2368
|
+
'clock', 'cutout', 'networkspeed', 'speed_container',
|
|
2369
|
+
'carrier', 'operator', 'sim_', 'mobile_signal'
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
# Token 优化:构建精简元素(只返回非空字段)
|
|
2373
|
+
def build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name):
|
|
2374
|
+
"""只返回有值的字段,节省 token"""
|
|
2375
|
+
item = {}
|
|
2376
|
+
if resource_id:
|
|
2377
|
+
# 精简 resource_id,只保留最后一段
|
|
2378
|
+
item['id'] = resource_id.split('/')[-1] if '/' in resource_id else resource_id
|
|
2379
|
+
if text:
|
|
2380
|
+
item['text'] = text
|
|
2381
|
+
if content_desc:
|
|
2382
|
+
item['desc'] = content_desc
|
|
2383
|
+
if bounds:
|
|
2384
|
+
item['bounds'] = bounds
|
|
2385
|
+
if likely_click:
|
|
2386
|
+
item['click'] = True # 启发式判断可点击
|
|
2387
|
+
# class 精简:只保留关键类型
|
|
2388
|
+
if class_name in ('EditText', 'TextInput', 'Button', 'ImageButton', 'CheckBox', 'Switch'):
|
|
2389
|
+
item['type'] = class_name
|
|
2390
|
+
return item
|
|
2391
|
+
|
|
2912
2392
|
result = []
|
|
2913
2393
|
for elem in elements:
|
|
2914
2394
|
# 获取元素属性
|
|
@@ -2926,16 +2406,19 @@ class BasicMobileToolsLite:
|
|
|
2926
2406
|
if bounds == '[0,0][0,0]':
|
|
2927
2407
|
continue
|
|
2928
2408
|
|
|
2409
|
+
# 1.5 过滤状态栏元素(对测试没有意义)
|
|
2410
|
+
if resource_id:
|
|
2411
|
+
resource_id_lower = resource_id.lower()
|
|
2412
|
+
if any(keyword in resource_id_lower for keyword in STATUS_BAR_KEYWORDS):
|
|
2413
|
+
continue
|
|
2414
|
+
|
|
2929
2415
|
# 2. 检查是否是功能控件(直接保留)
|
|
2930
2416
|
if class_name in FUNCTIONAL_WIDGETS:
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
'clickable': clickable,
|
|
2937
|
-
'class': class_name
|
|
2938
|
-
})
|
|
2417
|
+
# 使用启发式判断可点击性(替代不准确的 clickable 属性)
|
|
2418
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2419
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2420
|
+
if item:
|
|
2421
|
+
result.append(item)
|
|
2939
2422
|
continue
|
|
2940
2423
|
|
|
2941
2424
|
# 3. 检查是否是容器控件
|
|
@@ -2948,14 +2431,10 @@ class BasicMobileToolsLite:
|
|
|
2948
2431
|
# 所有属性都是默认值,过滤掉
|
|
2949
2432
|
continue
|
|
2950
2433
|
# 有业务ID或其他有意义属性,保留
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
'bounds': bounds,
|
|
2956
|
-
'clickable': clickable,
|
|
2957
|
-
'class': class_name
|
|
2958
|
-
})
|
|
2434
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2435
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2436
|
+
if item:
|
|
2437
|
+
result.append(item)
|
|
2959
2438
|
continue
|
|
2960
2439
|
|
|
2961
2440
|
# 4. 检查是否是装饰类控件
|
|
@@ -2972,19 +2451,72 @@ class BasicMobileToolsLite:
|
|
|
2972
2451
|
continue
|
|
2973
2452
|
|
|
2974
2453
|
# 6. 其他情况:有意义的元素保留
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2454
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2455
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2456
|
+
if item:
|
|
2457
|
+
result.append(item)
|
|
2458
|
+
|
|
2459
|
+
# Token 优化:可选限制返回元素数量(默认不限制,确保准确度)
|
|
2460
|
+
if TOKEN_OPTIMIZATION and MAX_ELEMENTS > 0 and len(result) > MAX_ELEMENTS:
|
|
2461
|
+
# 仅在用户明确设置 MAX_ELEMENTS_RETURN 时才截断
|
|
2462
|
+
truncated = result[:MAX_ELEMENTS]
|
|
2463
|
+
truncated.append({
|
|
2464
|
+
'_truncated': True,
|
|
2465
|
+
'_total': len(result),
|
|
2466
|
+
'_shown': MAX_ELEMENTS
|
|
2982
2467
|
})
|
|
2468
|
+
return truncated
|
|
2983
2469
|
|
|
2984
2470
|
return result
|
|
2985
2471
|
except Exception as e:
|
|
2986
2472
|
return [{"error": f"获取元素失败: {e}"}]
|
|
2987
2473
|
|
|
2474
|
+
def _get_page_texts(self, max_count: int = 15) -> List[str]:
|
|
2475
|
+
"""获取页面关键文本列表(用于点击后快速确认页面变化)
|
|
2476
|
+
|
|
2477
|
+
Args:
|
|
2478
|
+
max_count: 最多返回的文本数量
|
|
2479
|
+
|
|
2480
|
+
Returns:
|
|
2481
|
+
页面上的关键文本列表(去重)
|
|
2482
|
+
"""
|
|
2483
|
+
try:
|
|
2484
|
+
if self._is_ios():
|
|
2485
|
+
ios_client = self._get_ios_client()
|
|
2486
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
2487
|
+
# iOS: 获取所有 StaticText 的文本
|
|
2488
|
+
elements = ios_client.wda(type='XCUIElementTypeStaticText').find_elements()
|
|
2489
|
+
texts = set()
|
|
2490
|
+
for elem in elements[:50]: # 限制扫描数量
|
|
2491
|
+
try:
|
|
2492
|
+
name = elem.name or elem.label
|
|
2493
|
+
if name and len(name) > 1 and len(name) < 50:
|
|
2494
|
+
texts.add(name)
|
|
2495
|
+
except:
|
|
2496
|
+
pass
|
|
2497
|
+
return list(texts)[:max_count]
|
|
2498
|
+
return []
|
|
2499
|
+
else:
|
|
2500
|
+
# Android: 快速扫描 XML 获取文本
|
|
2501
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=True)
|
|
2502
|
+
import xml.etree.ElementTree as ET
|
|
2503
|
+
root = ET.fromstring(xml_string)
|
|
2504
|
+
|
|
2505
|
+
texts = set()
|
|
2506
|
+
for elem in root.iter():
|
|
2507
|
+
text = elem.get('text', '').strip()
|
|
2508
|
+
desc = elem.get('content-desc', '').strip()
|
|
2509
|
+
# 只收集有意义的文本(长度2-30,非纯数字)
|
|
2510
|
+
for t in [text, desc]:
|
|
2511
|
+
if t and 2 <= len(t) <= 30 and not t.isdigit():
|
|
2512
|
+
texts.add(t)
|
|
2513
|
+
if len(texts) >= max_count * 2: # 收集足够后停止
|
|
2514
|
+
break
|
|
2515
|
+
|
|
2516
|
+
return list(texts)[:max_count]
|
|
2517
|
+
except Exception:
|
|
2518
|
+
return []
|
|
2519
|
+
|
|
2988
2520
|
def _has_business_id(self, resource_id: str) -> bool:
|
|
2989
2521
|
"""
|
|
2990
2522
|
判断resource_id是否是业务相关的ID
|
|
@@ -3016,21 +2548,83 @@ class BasicMobileToolsLite:
|
|
|
3016
2548
|
|
|
3017
2549
|
return True
|
|
3018
2550
|
|
|
3019
|
-
def
|
|
3020
|
-
|
|
2551
|
+
def _is_likely_clickable(self, class_name: str, resource_id: str, text: str,
|
|
2552
|
+
content_desc: str, clickable: bool, bounds: str) -> bool:
|
|
2553
|
+
"""
|
|
2554
|
+
启发式判断元素是否可能可点击
|
|
3021
2555
|
|
|
3022
|
-
|
|
3023
|
-
|
|
2556
|
+
Android 的 clickable 属性经常不准确,因为:
|
|
2557
|
+
1. 点击事件可能设置在父容器上
|
|
2558
|
+
2. 使用 onTouchListener 而不是 onClick
|
|
2559
|
+
3. RecyclerView item 通过 ItemClickListener 处理
|
|
3024
2560
|
|
|
3025
|
-
|
|
3026
|
-
包含关闭按钮位置信息的字典,或截图让 AI 分析
|
|
2561
|
+
此方法通过多种规则推断元素的真实可点击性
|
|
3027
2562
|
"""
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
2563
|
+
# 规则1:clickable=true 肯定可点击
|
|
2564
|
+
if clickable:
|
|
2565
|
+
return True
|
|
2566
|
+
|
|
2567
|
+
# 规则2:特定类型的控件通常可点击
|
|
2568
|
+
TYPICALLY_CLICKABLE = {
|
|
2569
|
+
'Button', 'ImageButton', 'CheckBox', 'RadioButton', 'Switch',
|
|
2570
|
+
'ToggleButton', 'FloatingActionButton', 'Chip', 'TabView',
|
|
2571
|
+
'EditText', 'TextInput', # 输入框可点击获取焦点
|
|
2572
|
+
}
|
|
2573
|
+
if class_name in TYPICALLY_CLICKABLE:
|
|
2574
|
+
return True
|
|
2575
|
+
|
|
2576
|
+
# 规则3:resource_id 包含可点击关键词
|
|
2577
|
+
if resource_id:
|
|
2578
|
+
id_lower = resource_id.lower()
|
|
2579
|
+
CLICK_KEYWORDS = [
|
|
2580
|
+
'btn', 'button', 'click', 'tap', 'submit', 'confirm',
|
|
2581
|
+
'cancel', 'close', 'back', 'next', 'prev', 'more',
|
|
2582
|
+
'action', 'link', 'menu', 'tab', 'item', 'cell',
|
|
2583
|
+
'card', 'avatar', 'icon', 'entry', 'option', 'arrow'
|
|
2584
|
+
]
|
|
2585
|
+
for kw in CLICK_KEYWORDS:
|
|
2586
|
+
if kw in id_lower:
|
|
2587
|
+
return True
|
|
2588
|
+
|
|
2589
|
+
# 规则4:content_desc 包含可点击暗示
|
|
2590
|
+
if content_desc:
|
|
2591
|
+
desc_lower = content_desc.lower()
|
|
2592
|
+
CLICK_HINTS = ['点击', '按钮', '关闭', '返回', '更多', 'click', 'tap', 'button', 'close']
|
|
2593
|
+
for hint in CLICK_HINTS:
|
|
2594
|
+
if hint in desc_lower:
|
|
2595
|
+
return True
|
|
2596
|
+
|
|
2597
|
+
# 规则5:有 resource_id 或 content_desc 的小图标可能可点击
|
|
2598
|
+
# (纯 ImageView 不加判断,误判率太高)
|
|
2599
|
+
if class_name in ('ImageView', 'Image') and (resource_id or content_desc) and bounds:
|
|
2600
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
2601
|
+
if match:
|
|
2602
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2603
|
+
w, h = x2 - x1, y2 - y1
|
|
2604
|
+
# 小图标(20-100px)更可能是按钮
|
|
2605
|
+
if 20 <= w <= 100 and 20 <= h <= 100:
|
|
2606
|
+
return True
|
|
2607
|
+
|
|
2608
|
+
# 规则6:移除(TextView 误判率太高,只依赖上面的规则)
|
|
2609
|
+
# 如果有 clickable=true 或 ID/desc 中有关键词,前面的规则已经覆盖
|
|
2610
|
+
|
|
2611
|
+
return False
|
|
2612
|
+
|
|
2613
|
+
def find_close_button(self) -> Dict:
|
|
2614
|
+
"""智能查找关闭按钮(不点击,只返回位置)
|
|
2615
|
+
|
|
2616
|
+
从元素列表中找最可能的关闭按钮,返回其坐标和百分比位置。
|
|
2617
|
+
适用于关闭弹窗广告等场景。
|
|
2618
|
+
|
|
2619
|
+
Returns:
|
|
2620
|
+
包含关闭按钮位置信息的字典,或截图让 AI 分析
|
|
2621
|
+
"""
|
|
2622
|
+
try:
|
|
2623
|
+
import re
|
|
2624
|
+
|
|
2625
|
+
if self._is_ios():
|
|
2626
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2627
|
+
|
|
3034
2628
|
# 获取屏幕尺寸
|
|
3035
2629
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3036
2630
|
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
@@ -3040,6 +2634,14 @@ class BasicMobileToolsLite:
|
|
|
3040
2634
|
import xml.etree.ElementTree as ET
|
|
3041
2635
|
root = ET.fromstring(xml_string)
|
|
3042
2636
|
|
|
2637
|
+
# 🔴 先检测是否有弹窗,避免误识别普通页面的按钮
|
|
2638
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2639
|
+
root, screen_width, screen_height
|
|
2640
|
+
)
|
|
2641
|
+
|
|
2642
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
2643
|
+
return {"success": True, "popup": False}
|
|
2644
|
+
|
|
3043
2645
|
# 关闭按钮特征
|
|
3044
2646
|
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
|
|
3045
2647
|
candidates = []
|
|
@@ -3083,38 +2685,31 @@ class BasicMobileToolsLite:
|
|
|
3083
2685
|
|
|
3084
2686
|
# 策略3:小尺寸的 clickable 元素(可能是 X 图标)
|
|
3085
2687
|
elif clickable:
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
# 其他位置的小元素
|
|
3109
|
-
elif 'Image' in class_name:
|
|
3110
|
-
score = 50
|
|
3111
|
-
reason = f"图片元素 {width}x{height}px"
|
|
3112
|
-
else:
|
|
3113
|
-
score = 40
|
|
3114
|
-
reason = f"小型可点击元素 {width}x{height}px"
|
|
2688
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
2689
|
+
max_size = max(120, int(screen_width * 0.12))
|
|
2690
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2691
|
+
# 基于位置评分:角落位置加分
|
|
2692
|
+
rel_x = center_x / screen_width
|
|
2693
|
+
rel_y = center_y / screen_height
|
|
2694
|
+
|
|
2695
|
+
# 右上角得分最高
|
|
2696
|
+
if rel_x > 0.6 and rel_y < 0.5:
|
|
2697
|
+
score = 70 + (rel_x - 0.6) * 50 + (0.5 - rel_y) * 50
|
|
2698
|
+
reason = f"右上角小元素 {width}x{height}px"
|
|
2699
|
+
# 左上角
|
|
2700
|
+
elif rel_x < 0.4 and rel_y < 0.5:
|
|
2701
|
+
score = 60 + (0.4 - rel_x) * 50 + (0.5 - rel_y) * 50
|
|
2702
|
+
reason = f"左上角小元素 {width}x{height}px"
|
|
2703
|
+
# 其他位置的小元素
|
|
2704
|
+
elif 'Image' in class_name:
|
|
2705
|
+
score = 50
|
|
2706
|
+
reason = f"图片元素 {width}x{height}px"
|
|
2707
|
+
else:
|
|
2708
|
+
score = 40
|
|
2709
|
+
reason = f"小型可点击元素 {width}x{height}px"
|
|
3115
2710
|
|
|
3116
2711
|
if score > 0:
|
|
3117
|
-
|
|
2712
|
+
candidates.append({
|
|
3118
2713
|
'score': score,
|
|
3119
2714
|
'reason': reason,
|
|
3120
2715
|
'bounds': bounds_str,
|
|
@@ -3122,795 +2717,270 @@ class BasicMobileToolsLite:
|
|
|
3122
2717
|
'center_y': center_y,
|
|
3123
2718
|
'x_percent': x_percent,
|
|
3124
2719
|
'y_percent': y_percent,
|
|
3125
|
-
'size': f"{width}x{height}"
|
|
3126
|
-
|
|
3127
|
-
'content_desc': content_desc,
|
|
3128
|
-
'class': class_name,
|
|
3129
|
-
'clickable': clickable
|
|
3130
|
-
}
|
|
3131
|
-
# 尝试获取 resource-id(如果存在)
|
|
3132
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
3133
|
-
if resource_id:
|
|
3134
|
-
candidate['resource_id'] = resource_id
|
|
3135
|
-
candidates.append(candidate)
|
|
2720
|
+
'size': f"{width}x{height}"
|
|
2721
|
+
})
|
|
3136
2722
|
|
|
3137
2723
|
if not candidates:
|
|
3138
|
-
#
|
|
3139
|
-
|
|
2724
|
+
# 没找到,截图让 AI 分析
|
|
2725
|
+
screenshot_result = self.take_screenshot(description="找关闭按钮", compress=True)
|
|
3140
2726
|
return {
|
|
3141
2727
|
"success": False,
|
|
3142
|
-
"message": "❌
|
|
2728
|
+
"message": "❌ 元素树未找到关闭按钮,已截图供 AI 分析",
|
|
2729
|
+
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3143
2730
|
"screen_size": {"width": screen_width, "height": screen_height},
|
|
3144
|
-
"
|
|
2731
|
+
"image_size": {
|
|
2732
|
+
"width": screenshot_result.get("image_width"),
|
|
2733
|
+
"height": screenshot_result.get("image_height")
|
|
2734
|
+
},
|
|
2735
|
+
"original_size": {
|
|
2736
|
+
"width": screenshot_result.get("original_img_width"),
|
|
2737
|
+
"height": screenshot_result.get("original_img_height")
|
|
2738
|
+
},
|
|
2739
|
+
"tip": "请分析截图找到 X 关闭按钮,然后调用 mobile_click_by_percent(x_percent, y_percent)"
|
|
3145
2740
|
}
|
|
3146
2741
|
|
|
3147
2742
|
# 按得分排序
|
|
3148
2743
|
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3149
2744
|
best = candidates[0]
|
|
3150
2745
|
|
|
3151
|
-
#
|
|
3152
|
-
close_button_info = {
|
|
3153
|
-
"reason": best['reason'],
|
|
3154
|
-
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
3155
|
-
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
3156
|
-
"bounds": best['bounds'],
|
|
3157
|
-
"size": best['size'],
|
|
3158
|
-
"score": best['score']
|
|
3159
|
-
}
|
|
3160
|
-
|
|
3161
|
-
# 添加可用的点击方式(优化:优先使用 resource_id 或 text)
|
|
3162
|
-
if best.get('resource_id'):
|
|
3163
|
-
close_button_info['resource_id'] = best['resource_id']
|
|
3164
|
-
click_command = f"mobile_click_by_id('{best['resource_id']}')"
|
|
3165
|
-
elif best.get('text'):
|
|
3166
|
-
close_button_info['text'] = best['text']
|
|
3167
|
-
click_command = f"mobile_click_by_text('{best['text']}')"
|
|
3168
|
-
else:
|
|
3169
|
-
click_command = f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})"
|
|
3170
|
-
|
|
2746
|
+
# Token 优化:只返回最必要的信息
|
|
3171
2747
|
return {
|
|
3172
2748
|
"success": True,
|
|
3173
|
-
"
|
|
3174
|
-
"
|
|
3175
|
-
"
|
|
3176
|
-
"click_command": click_command,
|
|
3177
|
-
"other_candidates": [
|
|
3178
|
-
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
3179
|
-
for c in candidates[1:4]
|
|
3180
|
-
] if len(candidates) > 1 else [],
|
|
3181
|
-
"screen_size": {"width": screen_width, "height": screen_height}
|
|
2749
|
+
"popup": True,
|
|
2750
|
+
"close": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2751
|
+
"cmd": f"click_by_percent({best['x_percent']},{best['y_percent']})"
|
|
3182
2752
|
}
|
|
3183
2753
|
|
|
3184
2754
|
except Exception as e:
|
|
3185
|
-
return {"success": False, "
|
|
2755
|
+
return {"success": False, "msg": str(e)}
|
|
3186
2756
|
|
|
3187
|
-
def
|
|
3188
|
-
"""
|
|
2757
|
+
def close_popup(self) -> Dict:
|
|
2758
|
+
"""智能关闭弹窗(改进版)
|
|
3189
2759
|
|
|
3190
|
-
|
|
3191
|
-
包含 has_popup 和 popup_info 的字典
|
|
3192
|
-
"""
|
|
3193
|
-
try:
|
|
3194
|
-
if self._is_ios():
|
|
3195
|
-
return {"has_popup": False, "popup_info": None}
|
|
3196
|
-
|
|
3197
|
-
# 获取元素列表
|
|
3198
|
-
elements = self.list_elements()
|
|
3199
|
-
|
|
3200
|
-
# 分析页面内容
|
|
3201
|
-
page_analysis = self._detect_page_content(elements)
|
|
3202
|
-
|
|
3203
|
-
return {
|
|
3204
|
-
"has_popup": page_analysis.get("has_popup", False),
|
|
3205
|
-
"popup_info": page_analysis.get("popup_info")
|
|
3206
|
-
}
|
|
3207
|
-
except Exception:
|
|
3208
|
-
return {"has_popup": False, "popup_info": None}
|
|
3209
|
-
|
|
3210
|
-
def _detect_page_content(self, elements: List[Dict],
|
|
3211
|
-
confidence_threshold: float = None,
|
|
3212
|
-
xml_string: str = None) -> Dict:
|
|
3213
|
-
"""基于元素列表识别页面内容
|
|
2760
|
+
核心改进:先检测弹窗区域,再在弹窗范围内查找关闭按钮
|
|
3214
2761
|
|
|
3215
|
-
|
|
3216
|
-
|
|
2762
|
+
策略(优先级从高到低):
|
|
2763
|
+
1. 检测弹窗区域(非全屏的大面积容器)
|
|
2764
|
+
2. 在弹窗边界内查找关闭相关的文本/描述(×、X、关闭、close 等)
|
|
2765
|
+
3. 在弹窗边界内查找小尺寸的 clickable 元素(优先边角位置)
|
|
2766
|
+
4. 如果都找不到,截图让 AI 视觉识别
|
|
3217
2767
|
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
xml_string: 预先获取的 XML 字符串(可选,避免重复调用)
|
|
3222
|
-
|
|
3223
|
-
Returns:
|
|
3224
|
-
{
|
|
3225
|
-
"main_content": {...}, # 主要功能区域
|
|
3226
|
-
"has_popup": bool, # 是否有弹窗
|
|
3227
|
-
"popup_info": {...}, # 弹窗信息
|
|
3228
|
-
"interactive_elements": [...] # 可交互元素
|
|
3229
|
-
}
|
|
2768
|
+
适配策略:
|
|
2769
|
+
- X 按钮可能在任意位置(上下左右都支持)
|
|
2770
|
+
- 使用百分比坐标记录,跨分辨率兼容
|
|
3230
2771
|
"""
|
|
3231
|
-
if confidence_threshold is None:
|
|
3232
|
-
confidence_threshold = DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3233
|
-
|
|
3234
|
-
if self._is_ios():
|
|
3235
|
-
return self._detect_page_content_ios(elements, confidence_threshold)
|
|
3236
|
-
|
|
3237
|
-
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3238
|
-
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3239
|
-
|
|
3240
|
-
# 获取可交互元素
|
|
3241
|
-
interactive_elements = [
|
|
3242
|
-
elem for elem in elements
|
|
3243
|
-
if elem.get("clickable") is True or str(elem.get("clickable", "")).lower() == "true"
|
|
3244
|
-
]
|
|
3245
|
-
|
|
3246
|
-
# 分析是否有弹窗(基于元素特征)
|
|
3247
|
-
has_popup = False
|
|
3248
|
-
popup_info = None
|
|
3249
|
-
|
|
3250
|
-
# 通过XML检测弹窗(更准确)
|
|
3251
2772
|
try:
|
|
3252
|
-
|
|
3253
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2773
|
+
import re
|
|
3254
2774
|
import xml.etree.ElementTree as ET
|
|
3255
|
-
root = ET.fromstring(xml_string)
|
|
3256
|
-
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3257
|
-
root, screen_width, screen_height
|
|
3258
|
-
)
|
|
3259
|
-
if popup_bounds and popup_confidence >= confidence_threshold:
|
|
3260
|
-
has_popup = True
|
|
3261
|
-
popup_info = {
|
|
3262
|
-
"bounds": popup_bounds,
|
|
3263
|
-
"confidence": popup_confidence,
|
|
3264
|
-
"detected_by": "xml_structure"
|
|
3265
|
-
}
|
|
3266
|
-
except Exception:
|
|
3267
|
-
pass
|
|
3268
|
-
|
|
3269
|
-
# 如果没有通过XML检测到,检查是否有明显的弹窗按钮特征
|
|
3270
|
-
if not has_popup:
|
|
3271
|
-
for elem in interactive_elements:
|
|
3272
|
-
text = elem.get("text", "")
|
|
3273
|
-
resource_id = elem.get("resource_id", "")
|
|
3274
|
-
content_desc = elem.get("content_desc", "")
|
|
3275
|
-
|
|
3276
|
-
# 检查文本是否匹配确认按钮(优先检测)
|
|
3277
|
-
if text in CONFIRM_BUTTON_TEXTS:
|
|
3278
|
-
has_popup = True
|
|
3279
|
-
popup_info = {"detected_by": "confirm_button_text", "text": text}
|
|
3280
|
-
break
|
|
3281
|
-
|
|
3282
|
-
# 检查文本是否匹配关闭按钮
|
|
3283
|
-
if text in CLOSE_BUTTON_TEXTS:
|
|
3284
|
-
has_popup = True
|
|
3285
|
-
popup_info = {"detected_by": "close_button_text", "text": text}
|
|
3286
|
-
break
|
|
3287
|
-
|
|
3288
|
-
# 检查 resource-id 是否包含确认关键词
|
|
3289
|
-
if any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3290
|
-
has_popup = True
|
|
3291
|
-
popup_info = {"detected_by": "confirm_button_id", "resource_id": resource_id}
|
|
3292
|
-
break
|
|
3293
|
-
|
|
3294
|
-
# 检查 resource-id 是否包含关闭关键词
|
|
3295
|
-
if any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3296
|
-
has_popup = True
|
|
3297
|
-
popup_info = {"detected_by": "close_button_id", "resource_id": resource_id}
|
|
3298
|
-
break
|
|
3299
|
-
|
|
3300
|
-
# 检查 content-desc 是否包含确认关键词
|
|
3301
|
-
if any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3302
|
-
has_popup = True
|
|
3303
|
-
popup_info = {"detected_by": "confirm_button_desc", "content_desc": content_desc}
|
|
3304
|
-
break
|
|
3305
|
-
|
|
3306
|
-
# 检查 content-desc 是否包含关闭关键词
|
|
3307
|
-
if any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3308
|
-
has_popup = True
|
|
3309
|
-
popup_info = {"detected_by": "close_button_desc", "content_desc": content_desc}
|
|
3310
|
-
break
|
|
3311
|
-
|
|
3312
|
-
return {
|
|
3313
|
-
"has_popup": has_popup,
|
|
3314
|
-
"popup_info": popup_info,
|
|
3315
|
-
"interactive_elements": interactive_elements,
|
|
3316
|
-
"total_elements": len(elements)
|
|
3317
|
-
}
|
|
3318
|
-
|
|
3319
|
-
def _detect_page_content_ios(self, elements: List[Dict], confidence_threshold: float) -> Dict:
|
|
3320
|
-
"""iOS 平台基于元素列表识别页面内容
|
|
3321
|
-
|
|
3322
|
-
Args:
|
|
3323
|
-
elements: 元素列表
|
|
3324
|
-
confidence_threshold: 弹窗检测置信度阈值
|
|
3325
|
-
|
|
3326
|
-
Returns:
|
|
3327
|
-
页面内容分析结果
|
|
3328
|
-
"""
|
|
3329
|
-
# 获取可交互元素
|
|
3330
|
-
interactive_elements = [
|
|
3331
|
-
elem for elem in elements
|
|
3332
|
-
if elem.get("type") in ['XCUIElementTypeButton', 'XCUIElementTypeLink', 'XCUIElementTypeCell']
|
|
3333
|
-
]
|
|
3334
|
-
|
|
3335
|
-
has_popup = False
|
|
3336
|
-
popup_info = None
|
|
3337
|
-
|
|
3338
|
-
# iOS 弹窗检测:检查是否有 Alert/Sheet 类型元素
|
|
3339
|
-
for elem in elements:
|
|
3340
|
-
elem_type = elem.get("type", "")
|
|
3341
|
-
if elem_type in IOS_POPUP_TYPES:
|
|
3342
|
-
has_popup = True
|
|
3343
|
-
popup_info = {
|
|
3344
|
-
"detected_by": "ios_popup_type",
|
|
3345
|
-
"type": elem_type,
|
|
3346
|
-
"bounds": elem.get("bounds")
|
|
3347
|
-
}
|
|
3348
|
-
break
|
|
3349
|
-
|
|
3350
|
-
# 如果没有检测到系统弹窗,检查是否有确认/关闭按钮文本
|
|
3351
|
-
if not has_popup:
|
|
3352
|
-
for elem in interactive_elements:
|
|
3353
|
-
text = elem.get("label", "") or elem.get("text", "") or ""
|
|
3354
|
-
# 优先检测确认按钮
|
|
3355
|
-
if text in CONFIRM_BUTTON_TEXTS:
|
|
3356
|
-
has_popup = True
|
|
3357
|
-
popup_info = {"detected_by": "confirm_button_text", "text": text}
|
|
3358
|
-
break
|
|
3359
|
-
# 其次检测关闭按钮
|
|
3360
|
-
if text in CLOSE_BUTTON_TEXTS:
|
|
3361
|
-
has_popup = True
|
|
3362
|
-
popup_info = {"detected_by": "close_button_text", "text": text}
|
|
3363
|
-
break
|
|
3364
|
-
|
|
3365
|
-
return {
|
|
3366
|
-
"has_popup": has_popup,
|
|
3367
|
-
"popup_info": popup_info,
|
|
3368
|
-
"interactive_elements": interactive_elements,
|
|
3369
|
-
"total_elements": len(elements)
|
|
3370
|
-
}
|
|
3371
|
-
|
|
3372
|
-
def _find_close_button_in_elements(self, elements: List[Dict], page_analysis: Dict = None) -> Optional[Dict]:
|
|
3373
|
-
"""在元素列表中查找关闭按钮
|
|
3374
|
-
|
|
3375
|
-
优先级顺序:
|
|
3376
|
-
1. 确认性按钮(同意、确认、允许等)- 最高优先级
|
|
3377
|
-
2. 关闭/取消按钮(关闭、取消、跳过等)- 次优先级
|
|
3378
|
-
3. 小尺寸可点击元素(可能是 X 图标)- 最低优先级
|
|
3379
|
-
|
|
3380
|
-
使用全局常量 CONFIRM_BUTTON_TEXTS、CLOSE_BUTTON_TEXTS、
|
|
3381
|
-
CONFIRM_BUTTON_KEYWORDS、CLOSE_BUTTON_KEYWORDS、EXCLUDE_ELEMENT_KEYWORDS
|
|
3382
|
-
|
|
3383
|
-
Args:
|
|
3384
|
-
elements: 元素列表
|
|
3385
|
-
page_analysis: 页面分析结果(可选)
|
|
3386
|
-
|
|
3387
|
-
Returns:
|
|
3388
|
-
关闭按钮信息,如果未找到返回None
|
|
3389
|
-
"""
|
|
3390
|
-
if self._is_ios():
|
|
3391
|
-
return self._find_close_button_in_elements_ios(elements)
|
|
3392
|
-
|
|
3393
|
-
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3394
|
-
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3395
|
-
|
|
3396
|
-
candidates = []
|
|
3397
|
-
|
|
3398
|
-
for elem in elements:
|
|
3399
|
-
text = elem.get("text", "").strip()
|
|
3400
|
-
content_desc = elem.get("content_desc", "").strip()
|
|
3401
|
-
resource_id = elem.get("resource_id", "").strip()
|
|
3402
|
-
bounds = elem.get("bounds", "")
|
|
3403
|
-
clickable = elem.get("clickable", False)
|
|
3404
|
-
|
|
3405
|
-
if not bounds:
|
|
3406
|
-
continue
|
|
3407
|
-
|
|
3408
|
-
# 解析 bounds
|
|
3409
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
3410
|
-
if not match:
|
|
3411
|
-
continue
|
|
3412
|
-
|
|
3413
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
3414
|
-
width = x2 - x1
|
|
3415
|
-
height = y2 - y1
|
|
3416
|
-
center_x = (x1 + x2) // 2
|
|
3417
|
-
center_y = (y1 + y2) // 2
|
|
3418
|
-
|
|
3419
|
-
# 计算百分比
|
|
3420
|
-
x_percent = round(center_x / screen_width * 100, 1)
|
|
3421
|
-
y_percent = round(center_y / screen_height * 100, 1)
|
|
3422
|
-
|
|
3423
|
-
score = 0
|
|
3424
|
-
reason = ""
|
|
3425
|
-
|
|
3426
|
-
# 策略1:确认性按钮文本(最高优先级)- 优先点击同意、确认等
|
|
3427
|
-
if text in CONFIRM_BUTTON_TEXTS:
|
|
3428
|
-
score = 120 # 最高分
|
|
3429
|
-
reason = f"确认按钮文本='{text}'"
|
|
3430
|
-
|
|
3431
|
-
# 策略2:确认类 resource-id 匹配
|
|
3432
|
-
elif any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3433
|
-
score = 115
|
|
3434
|
-
reason = f"确认按钮ID='{resource_id}'"
|
|
3435
|
-
|
|
3436
|
-
# 策略3:确认类 content-desc 匹配
|
|
3437
|
-
elif any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3438
|
-
score = 110
|
|
3439
|
-
reason = f"确认按钮描述='{content_desc}'"
|
|
3440
|
-
|
|
3441
|
-
# 策略4:关闭/取消按钮文本(次优先级)
|
|
3442
|
-
elif text in CLOSE_BUTTON_TEXTS:
|
|
3443
|
-
score = 100
|
|
3444
|
-
reason = f"关闭按钮文本='{text}'"
|
|
3445
|
-
|
|
3446
|
-
# 策略5:关闭类 resource-id 匹配
|
|
3447
|
-
elif any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3448
|
-
score = 95
|
|
3449
|
-
reason = f"关闭按钮ID='{resource_id}'"
|
|
3450
|
-
|
|
3451
|
-
# 策略6:关闭类 content-desc 匹配
|
|
3452
|
-
elif any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3453
|
-
score = 90
|
|
3454
|
-
reason = f"关闭按钮描述='{content_desc}'"
|
|
3455
|
-
|
|
3456
|
-
# 策略7:小尺寸的 clickable 元素(可能是 X 图标)- 最低优先级
|
|
3457
|
-
elif clickable:
|
|
3458
|
-
# 排除非关闭按钮的关键词 - 使用全局常量
|
|
3459
|
-
if any(kw in resource_id.lower() for kw in EXCLUDE_ELEMENT_KEYWORDS):
|
|
3460
|
-
score = 0 # 排除这些元素
|
|
3461
|
-
elif any(kw in content_desc.lower() for kw in EXCLUDE_ELEMENT_KEYWORDS):
|
|
3462
|
-
score = 0 # 排除这些元素
|
|
3463
|
-
else:
|
|
3464
|
-
min_size = max(20, int(screen_width * 0.03))
|
|
3465
|
-
max_size = max(120, int(screen_width * 0.15))
|
|
3466
|
-
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
3467
|
-
rel_x = center_x / screen_width
|
|
3468
|
-
rel_y = center_y / screen_height
|
|
3469
|
-
|
|
3470
|
-
# 右上角得分最高
|
|
3471
|
-
if rel_x > 0.6 and rel_y < 0.5:
|
|
3472
|
-
score = 70 + (rel_x - 0.6) * 50 + (0.5 - rel_y) * 50
|
|
3473
|
-
reason = f"右上角小元素 {width}x{height}px"
|
|
3474
|
-
# 左上角
|
|
3475
|
-
elif rel_x < 0.4 and rel_y < 0.5:
|
|
3476
|
-
score = 60 + (0.4 - rel_x) * 50 + (0.5 - rel_y) * 50
|
|
3477
|
-
reason = f"左上角小元素 {width}x{height}px"
|
|
3478
|
-
else:
|
|
3479
|
-
score = 40
|
|
3480
|
-
reason = f"小型可点击元素 {width}x{height}px"
|
|
3481
|
-
|
|
3482
|
-
if score > 0:
|
|
3483
|
-
candidates.append({
|
|
3484
|
-
'score': score,
|
|
3485
|
-
'reason': reason,
|
|
3486
|
-
'bounds': bounds,
|
|
3487
|
-
'center_x': center_x,
|
|
3488
|
-
'center_y': center_y,
|
|
3489
|
-
'x_percent': x_percent,
|
|
3490
|
-
'y_percent': y_percent,
|
|
3491
|
-
'text': text,
|
|
3492
|
-
'resource_id': resource_id,
|
|
3493
|
-
'content_desc': content_desc
|
|
3494
|
-
})
|
|
3495
|
-
|
|
3496
|
-
if not candidates:
|
|
3497
|
-
return None
|
|
3498
|
-
|
|
3499
|
-
# 按得分排序,返回最佳候选(确认按钮优先)
|
|
3500
|
-
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3501
|
-
return candidates[0]
|
|
3502
|
-
|
|
3503
|
-
def _find_close_button_in_elements_ios(self, elements: List[Dict]) -> Optional[Dict]:
|
|
3504
|
-
"""iOS 平台在元素列表中查找关闭按钮
|
|
3505
|
-
|
|
3506
|
-
优先级顺序:
|
|
3507
|
-
1. 确认性按钮(同意、确认、允许等)- 最高优先级
|
|
3508
|
-
2. 关闭/取消按钮(关闭、取消、跳过等)- 次优先级
|
|
3509
|
-
3. 小尺寸可点击元素(可能是 X 图标)- 最低优先级
|
|
3510
|
-
|
|
3511
|
-
Args:
|
|
3512
|
-
elements: 元素列表
|
|
3513
|
-
|
|
3514
|
-
Returns:
|
|
3515
|
-
关闭按钮信息,如果未找到返回None
|
|
3516
|
-
"""
|
|
3517
|
-
ios_client = self._get_ios_client()
|
|
3518
|
-
if not ios_client:
|
|
3519
|
-
return None
|
|
3520
|
-
|
|
3521
|
-
try:
|
|
3522
|
-
screen_size = ios_client.window_size()
|
|
3523
|
-
screen_width = screen_size.width
|
|
3524
|
-
screen_height = screen_size.height
|
|
3525
|
-
except:
|
|
3526
|
-
screen_width = 375
|
|
3527
|
-
screen_height = 812
|
|
3528
|
-
|
|
3529
|
-
candidates = []
|
|
3530
|
-
|
|
3531
|
-
for elem in elements:
|
|
3532
|
-
text = elem.get("text", "") or elem.get("label", "") or ""
|
|
3533
|
-
text = text.strip()
|
|
3534
|
-
elem_type = elem.get("type", "")
|
|
3535
|
-
bounds = elem.get("bounds", {})
|
|
3536
|
-
|
|
3537
|
-
# iOS bounds 格式不同
|
|
3538
|
-
if isinstance(bounds, dict):
|
|
3539
|
-
x = bounds.get("x", 0)
|
|
3540
|
-
y = bounds.get("y", 0)
|
|
3541
|
-
width = bounds.get("width", 0)
|
|
3542
|
-
height = bounds.get("height", 0)
|
|
3543
|
-
else:
|
|
3544
|
-
continue
|
|
3545
2775
|
|
|
3546
|
-
|
|
3547
|
-
center_y = int(y + height / 2)
|
|
3548
|
-
x_percent = round(center_x / screen_width * 100, 1)
|
|
3549
|
-
y_percent = round(center_y / screen_height * 100, 1)
|
|
3550
|
-
|
|
3551
|
-
score = 0
|
|
3552
|
-
reason = ""
|
|
3553
|
-
|
|
3554
|
-
# 策略1:确认性按钮文本(最高优先级)
|
|
3555
|
-
if text in CONFIRM_BUTTON_TEXTS:
|
|
3556
|
-
score = 120
|
|
3557
|
-
reason = f"确认按钮文本='{text}'"
|
|
3558
|
-
|
|
3559
|
-
# 策略2:确认类关键词匹配
|
|
3560
|
-
elif any(kw in text.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3561
|
-
score = 115
|
|
3562
|
-
reason = f"确认按钮关键词: '{text}'"
|
|
3563
|
-
|
|
3564
|
-
# 策略3:关闭/取消按钮文本(次优先级)
|
|
3565
|
-
elif text in CLOSE_BUTTON_TEXTS:
|
|
3566
|
-
score = 100
|
|
3567
|
-
reason = f"关闭按钮文本='{text}'"
|
|
3568
|
-
|
|
3569
|
-
# 策略4:关闭类关键词匹配
|
|
3570
|
-
elif any(kw in text.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3571
|
-
score = 95
|
|
3572
|
-
reason = f"关闭按钮关键词: '{text}'"
|
|
3573
|
-
|
|
3574
|
-
# 策略5:XCUIElementTypeButton 类型的小元素(最低优先级)
|
|
3575
|
-
elif elem_type == 'XCUIElementTypeButton':
|
|
3576
|
-
if 20 <= width <= 60 and 20 <= height <= 60:
|
|
3577
|
-
rel_x = center_x / screen_width
|
|
3578
|
-
rel_y = center_y / screen_height
|
|
3579
|
-
if rel_x > 0.7 and rel_y < 0.3:
|
|
3580
|
-
score = 70
|
|
3581
|
-
reason = f"右上角小按钮 {width}x{height}px"
|
|
3582
|
-
|
|
3583
|
-
if score > 0:
|
|
3584
|
-
candidates.append({
|
|
3585
|
-
'score': score,
|
|
3586
|
-
'reason': reason,
|
|
3587
|
-
'bounds': f"[{x},{y}][{x+width},{y+height}]",
|
|
3588
|
-
'center_x': center_x,
|
|
3589
|
-
'center_y': center_y,
|
|
3590
|
-
'x_percent': x_percent,
|
|
3591
|
-
'y_percent': y_percent,
|
|
3592
|
-
'text': text,
|
|
3593
|
-
'resource_id': '',
|
|
3594
|
-
'content_desc': ''
|
|
3595
|
-
})
|
|
3596
|
-
|
|
3597
|
-
if not candidates:
|
|
3598
|
-
return None
|
|
3599
|
-
|
|
3600
|
-
# 按得分排序(确认按钮优先)
|
|
3601
|
-
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3602
|
-
return candidates[0]
|
|
3603
|
-
|
|
3604
|
-
def _close_popup_by_elements(self, elements: List[Dict] = None,
|
|
3605
|
-
xml_string: str = None,
|
|
3606
|
-
confidence_threshold: float = None,
|
|
3607
|
-
page_fingerprint_before: Dict = None) -> Dict:
|
|
3608
|
-
"""阶段1:通过控件树关闭弹窗(最快、最可靠)
|
|
3609
|
-
|
|
3610
|
-
Args:
|
|
3611
|
-
elements: 预获取的元素列表(可选,避免重复调用)
|
|
3612
|
-
xml_string: 预获取的 XML 字符串(可选,避免重复调用)
|
|
3613
|
-
confidence_threshold: 弹窗检测置信度阈值
|
|
3614
|
-
page_fingerprint_before: 操作前的页面指纹(用于验证)
|
|
3615
|
-
|
|
3616
|
-
Returns:
|
|
3617
|
-
关闭结果
|
|
3618
|
-
"""
|
|
3619
|
-
try:
|
|
3620
|
-
if self._is_ios():
|
|
3621
|
-
return {"success": False, "reason": "iOS 请使用 _close_popup_ios"}
|
|
3622
|
-
|
|
3623
|
-
if confidence_threshold is None:
|
|
3624
|
-
confidence_threshold = DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3625
|
-
|
|
3626
|
-
# 1. 获取元素列表(复用或新获取)
|
|
3627
|
-
if elements is None:
|
|
3628
|
-
elements = self.list_elements()
|
|
3629
|
-
|
|
3630
|
-
# 2. 分析页面内容(传入预获取的 XML)
|
|
3631
|
-
page_analysis = self._detect_page_content(
|
|
3632
|
-
elements,
|
|
3633
|
-
confidence_threshold=confidence_threshold,
|
|
3634
|
-
xml_string=xml_string
|
|
3635
|
-
)
|
|
3636
|
-
|
|
3637
|
-
# 3. 如果没有弹窗,直接返回
|
|
3638
|
-
if not page_analysis.get("has_popup"):
|
|
3639
|
-
return {"success": False, "reason": "未检测到弹窗", "method": "elements"}
|
|
3640
|
-
|
|
3641
|
-
# 4. 查找关闭按钮
|
|
3642
|
-
close_button = self._find_close_button_in_elements(elements, page_analysis)
|
|
3643
|
-
|
|
3644
|
-
# 5. 如果找到,点击
|
|
3645
|
-
if close_button:
|
|
3646
|
-
center_x = close_button['center_x']
|
|
3647
|
-
center_y = close_button['center_y']
|
|
3648
|
-
|
|
3649
|
-
self.client.u2.click(center_x, center_y)
|
|
3650
|
-
time.sleep(0.5)
|
|
3651
|
-
|
|
3652
|
-
# 检查应用是否跳转
|
|
3653
|
-
app_check = self._check_app_switched()
|
|
3654
|
-
return_result = None
|
|
3655
|
-
if app_check['switched']:
|
|
3656
|
-
return_result = self._return_to_target_app()
|
|
3657
|
-
|
|
3658
|
-
# 【优化】使用页面指纹对比验证
|
|
3659
|
-
verified = self._verify_popup_closed_with_fingerprint(page_fingerprint_before)
|
|
3660
|
-
|
|
3661
|
-
msg = f"✅ 通过控件树找到关闭按钮并点击\n"
|
|
3662
|
-
msg += f" 原因: {close_button['reason']}\n"
|
|
3663
|
-
msg += f" 位置: ({center_x}, {center_y}) [({close_button['x_percent']}%, {close_button['y_percent']}%)]"
|
|
3664
|
-
if close_button.get('text'):
|
|
3665
|
-
msg += f"\n 文本: {close_button['text']}"
|
|
3666
|
-
if close_button.get('resource_id'):
|
|
3667
|
-
msg += f"\n ID: {close_button['resource_id']}"
|
|
3668
|
-
|
|
3669
|
-
if app_check['switched']:
|
|
3670
|
-
msg += f"\n⚠️ 应用已跳转"
|
|
3671
|
-
if return_result and return_result.get('success'):
|
|
3672
|
-
msg += f"\n{return_result['message']}"
|
|
3673
|
-
|
|
3674
|
-
if not verified:
|
|
3675
|
-
msg += f"\n⚠️ 验证失败:弹窗可能未完全关闭"
|
|
3676
|
-
|
|
3677
|
-
return {
|
|
3678
|
-
"success": verified,
|
|
3679
|
-
"method": "elements",
|
|
3680
|
-
"message": msg,
|
|
3681
|
-
"clicked": {
|
|
3682
|
-
"center": (center_x, center_y),
|
|
3683
|
-
"percent": (close_button['x_percent'], close_button['y_percent']),
|
|
3684
|
-
"reason": close_button['reason']
|
|
3685
|
-
},
|
|
3686
|
-
"app_check": app_check,
|
|
3687
|
-
"return_to_app": return_result,
|
|
3688
|
-
"verified": verified
|
|
3689
|
-
}
|
|
3690
|
-
|
|
3691
|
-
return {"success": False, "reason": "控件树中未找到关闭按钮", "method": "elements"}
|
|
3692
|
-
|
|
3693
|
-
except Exception as e:
|
|
3694
|
-
return {"success": False, "reason": f"控件树方法失败: {e}", "method": "elements"}
|
|
3695
|
-
|
|
3696
|
-
def _close_popup_by_screenshot_ai(self, elements: List[Dict] = None) -> Dict:
|
|
3697
|
-
"""阶段2:通过截图AI分析关闭弹窗(仅在控件树失败时)
|
|
3698
|
-
|
|
3699
|
-
【优化】提供关闭按钮候选建议,形成闭环
|
|
3700
|
-
|
|
3701
|
-
Args:
|
|
3702
|
-
elements: 预获取的元素列表(可选)
|
|
3703
|
-
|
|
3704
|
-
Returns:
|
|
3705
|
-
关闭结果(包含候选建议,需要AI分析后确定最终操作)
|
|
3706
|
-
"""
|
|
3707
|
-
try:
|
|
2776
|
+
# 获取屏幕尺寸
|
|
3708
2777
|
if self._is_ios():
|
|
3709
|
-
return {"success": False, "
|
|
3710
|
-
|
|
3711
|
-
# 截图(带SoM标注)
|
|
3712
|
-
som_result = self.take_screenshot_with_som()
|
|
3713
|
-
|
|
3714
|
-
# 【优化】基于元素列表提供关闭按钮候选建议
|
|
3715
|
-
close_button_candidates = []
|
|
3716
|
-
if elements is None:
|
|
3717
|
-
elements = self.list_elements()
|
|
3718
|
-
|
|
3719
|
-
# 查找所有可能的关闭按钮
|
|
3720
|
-
for elem in elements:
|
|
3721
|
-
text = elem.get("text", "").strip()
|
|
3722
|
-
resource_id = elem.get("resource_id", "").strip()
|
|
3723
|
-
content_desc = elem.get("content_desc", "").strip()
|
|
3724
|
-
bounds = elem.get("bounds", "")
|
|
3725
|
-
|
|
3726
|
-
if not bounds:
|
|
3727
|
-
continue
|
|
3728
|
-
|
|
3729
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
3730
|
-
if not match:
|
|
3731
|
-
continue
|
|
3732
|
-
|
|
3733
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
3734
|
-
center_x = (x1 + x2) // 2
|
|
3735
|
-
center_y = (y1 + y2) // 2
|
|
3736
|
-
|
|
3737
|
-
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
3738
|
-
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3739
|
-
x_percent = round(center_x / screen_width * 100, 1)
|
|
3740
|
-
y_percent = round(center_y / screen_height * 100, 1)
|
|
3741
|
-
|
|
3742
|
-
score = 0
|
|
3743
|
-
reason = ""
|
|
3744
|
-
|
|
3745
|
-
# 检查是否是确认按钮(优先)
|
|
3746
|
-
if text in CONFIRM_BUTTON_TEXTS:
|
|
3747
|
-
score = 120
|
|
3748
|
-
reason = f"确认按钮文本='{text}'"
|
|
3749
|
-
elif any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3750
|
-
score = 115
|
|
3751
|
-
reason = f"确认按钮ID: {resource_id}"
|
|
3752
|
-
elif any(kw in content_desc.lower() for kw in CONFIRM_BUTTON_KEYWORDS):
|
|
3753
|
-
score = 110
|
|
3754
|
-
reason = f"确认按钮描述: {content_desc}"
|
|
3755
|
-
# 检查是否是关闭按钮(次优先)
|
|
3756
|
-
elif text in CLOSE_BUTTON_TEXTS:
|
|
3757
|
-
score = 100
|
|
3758
|
-
reason = f"关闭按钮文本='{text}'"
|
|
3759
|
-
elif any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3760
|
-
score = 95
|
|
3761
|
-
reason = f"关闭按钮ID: {resource_id}"
|
|
3762
|
-
elif any(kw in content_desc.lower() for kw in CLOSE_BUTTON_KEYWORDS):
|
|
3763
|
-
score = 90
|
|
3764
|
-
reason = f"关闭按钮描述: {content_desc}"
|
|
3765
|
-
|
|
3766
|
-
if score > 0:
|
|
3767
|
-
# 确定推荐操作方式
|
|
3768
|
-
if text in CONFIRM_BUTTON_TEXTS or text in CLOSE_BUTTON_TEXTS:
|
|
3769
|
-
suggested_action = f"mobile_click_by_text('{text}')"
|
|
3770
|
-
else:
|
|
3771
|
-
suggested_action = f"mobile_click_by_percent({x_percent}, {y_percent})"
|
|
3772
|
-
|
|
3773
|
-
close_button_candidates.append({
|
|
3774
|
-
'score': score,
|
|
3775
|
-
'reason': reason,
|
|
3776
|
-
'text': text,
|
|
3777
|
-
'resource_id': resource_id,
|
|
3778
|
-
'center_x': center_x,
|
|
3779
|
-
'center_y': center_y,
|
|
3780
|
-
'x_percent': x_percent,
|
|
3781
|
-
'y_percent': y_percent,
|
|
3782
|
-
'suggested_action': suggested_action
|
|
3783
|
-
})
|
|
3784
|
-
|
|
3785
|
-
# 按分数排序
|
|
3786
|
-
close_button_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2778
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
3787
2779
|
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
"success": False, # 需要AI分析后才能确定
|
|
3791
|
-
"method": "screenshot_ai",
|
|
3792
|
-
"message": "已截图,等待AI分析",
|
|
3793
|
-
"screenshot": som_result.get("screenshot_path"),
|
|
3794
|
-
"elements": som_result.get("elements", []),
|
|
3795
|
-
"popup_detected": som_result.get("popup_detected", False),
|
|
3796
|
-
"close_button_candidates": close_button_candidates[:5], # 返回前5个候选
|
|
3797
|
-
"next_step": "AI分析截图后,根据分析结果调用对应的点击方法"
|
|
3798
|
-
}
|
|
2780
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2781
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
3799
2782
|
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
def _close_popup_by_template(self, page_fingerprint_before: Dict = None) -> Dict:
|
|
3804
|
-
"""阶段3:通过模板匹配关闭弹窗(最精确的兜底方案)
|
|
3805
|
-
|
|
3806
|
-
Args:
|
|
3807
|
-
page_fingerprint_before: 操作前的页面指纹(用于验证)
|
|
2783
|
+
# 获取原始 XML
|
|
2784
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
3808
2785
|
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
try:
|
|
3813
|
-
if self._is_ios():
|
|
3814
|
-
return {"success": False, "reason": "iOS 暂不支持"}
|
|
2786
|
+
# 关闭按钮的文本特征
|
|
2787
|
+
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', 'CLOSE', '跳过', '知道了']
|
|
2788
|
+
close_desc_keywords = ['关闭', 'close', 'dismiss', 'cancel', '跳过']
|
|
3815
2789
|
|
|
3816
|
-
|
|
3817
|
-
|
|
2790
|
+
close_candidates = []
|
|
2791
|
+
popup_bounds = None # 弹窗区域
|
|
3818
2792
|
|
|
3819
|
-
#
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
self.client.u2.click(x, y)
|
|
3825
|
-
time.sleep(0.5)
|
|
3826
|
-
|
|
3827
|
-
# 检查应用是否跳转
|
|
3828
|
-
app_check = self._check_app_switched()
|
|
3829
|
-
return_result = None
|
|
3830
|
-
if app_check['switched']:
|
|
3831
|
-
return_result = self._return_to_target_app()
|
|
3832
|
-
|
|
3833
|
-
# 【优化】使用页面指纹对比验证
|
|
3834
|
-
verified = self._verify_popup_closed_with_fingerprint(page_fingerprint_before)
|
|
2793
|
+
# 解析 XML
|
|
2794
|
+
try:
|
|
2795
|
+
root = ET.fromstring(xml_string)
|
|
2796
|
+
all_elements = list(root.iter())
|
|
3835
2797
|
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
2798
|
+
# ===== 第一步:使用严格的置信度检测弹窗区域 =====
|
|
2799
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2800
|
+
root, screen_width, screen_height
|
|
2801
|
+
)
|
|
3840
2802
|
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
if return_result and return_result.get('success'):
|
|
3844
|
-
msg += f"\n{return_result['message']}"
|
|
2803
|
+
# 如果置信度不够高,记录但继续尝试查找关闭按钮
|
|
2804
|
+
popup_detected = popup_bounds is not None and popup_confidence >= 0.6
|
|
3845
2805
|
|
|
3846
|
-
|
|
3847
|
-
|
|
2806
|
+
# 🔴 关键检查:如果没有检测到弹窗区域,直接返回"无弹窗"
|
|
2807
|
+
# 避免误点击普通页面上的"关闭"、"取消"等按钮
|
|
2808
|
+
if not popup_detected:
|
|
2809
|
+
return {"success": True, "popup": False}
|
|
3848
2810
|
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
2811
|
+
# ===== 第二步:在弹窗范围内查找关闭按钮 =====
|
|
2812
|
+
for idx, elem in enumerate(all_elements):
|
|
2813
|
+
text = elem.attrib.get('text', '')
|
|
2814
|
+
content_desc = elem.attrib.get('content-desc', '')
|
|
2815
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2816
|
+
class_name = elem.attrib.get('class', '')
|
|
2817
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2818
|
+
|
|
2819
|
+
if not bounds_str:
|
|
2820
|
+
continue
|
|
2821
|
+
|
|
2822
|
+
# 解析 bounds
|
|
2823
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2824
|
+
if not match:
|
|
2825
|
+
continue
|
|
2826
|
+
|
|
2827
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2828
|
+
width = x2 - x1
|
|
2829
|
+
height = y2 - y1
|
|
2830
|
+
center_x = (x1 + x2) // 2
|
|
2831
|
+
center_y = (y1 + y2) // 2
|
|
2832
|
+
|
|
2833
|
+
# 如果检测到弹窗区域,检查元素是否在弹窗范围内或附近
|
|
2834
|
+
in_popup = True
|
|
2835
|
+
popup_edge_bonus = 0
|
|
2836
|
+
is_floating_close = False # 是否是浮动关闭按钮(在弹窗外部上方)
|
|
2837
|
+
if popup_bounds:
|
|
2838
|
+
px1, py1, px2, py2 = popup_bounds
|
|
2839
|
+
|
|
2840
|
+
# 关闭按钮可能在弹窗外部(常见设计:X 按钮浮在弹窗右上角外侧)
|
|
2841
|
+
# 扩大搜索范围:弹窗上方 200 像素,右侧 50 像素
|
|
2842
|
+
margin_top = 200 # 上方扩展范围(关闭按钮常在弹窗上方)
|
|
2843
|
+
margin_side = 50 # 左右扩展范围
|
|
2844
|
+
margin_bottom = 30 # 下方扩展范围
|
|
2845
|
+
|
|
2846
|
+
in_popup = (px1 - margin_side <= center_x <= px2 + margin_side and
|
|
2847
|
+
py1 - margin_top <= center_y <= py2 + margin_bottom)
|
|
2848
|
+
|
|
2849
|
+
# 检查是否是浮动关闭按钮(在弹窗外侧:上方或下方)
|
|
2850
|
+
# 上方浮动关闭按钮(常见:右上角外侧)
|
|
2851
|
+
if center_y < py1 and center_y > py1 - margin_top:
|
|
2852
|
+
if center_x > (px1 + px2) / 2: # 在弹窗右半部分上方
|
|
2853
|
+
is_floating_close = True
|
|
2854
|
+
# 下方浮动关闭按钮(常见:底部中间外侧)
|
|
2855
|
+
elif center_y > py2 and center_y < py2 + margin_top:
|
|
2856
|
+
# 下方关闭按钮通常在中间位置
|
|
2857
|
+
if abs(center_x - (px1 + px2) / 2) < (px2 - px1) / 2:
|
|
2858
|
+
is_floating_close = True
|
|
2859
|
+
|
|
2860
|
+
if in_popup:
|
|
2861
|
+
# 计算元素是否在弹窗边缘(关闭按钮通常在边缘)
|
|
2862
|
+
dist_to_top = abs(center_y - py1)
|
|
2863
|
+
dist_to_bottom = abs(center_y - py2)
|
|
2864
|
+
dist_to_left = abs(center_x - px1)
|
|
2865
|
+
dist_to_right = abs(center_x - px2)
|
|
2866
|
+
min_dist = min(dist_to_top, dist_to_bottom, dist_to_left, dist_to_right)
|
|
2867
|
+
|
|
2868
|
+
# 在弹窗边缘 100 像素内的元素加分
|
|
2869
|
+
if min_dist < 100:
|
|
2870
|
+
popup_edge_bonus = 3.0 * (1 - min_dist / 100)
|
|
2871
|
+
|
|
2872
|
+
# 浮动关闭按钮(在弹窗上方外侧)给予高额加分
|
|
2873
|
+
if is_floating_close:
|
|
2874
|
+
popup_edge_bonus += 5.0 # 大幅加分
|
|
2875
|
+
|
|
2876
|
+
if not in_popup:
|
|
2877
|
+
continue
|
|
2878
|
+
|
|
2879
|
+
# 相对位置(0-1)
|
|
2880
|
+
rel_x = center_x / screen_width
|
|
2881
|
+
rel_y = center_y / screen_height
|
|
2882
|
+
|
|
2883
|
+
score = 0
|
|
2884
|
+
match_type = ""
|
|
2885
|
+
position = self._get_position_name(rel_x, rel_y)
|
|
2886
|
+
|
|
2887
|
+
# ===== 策略1:精确匹配关闭文本(最高优先级)=====
|
|
2888
|
+
if text in close_texts:
|
|
2889
|
+
score = 15.0 + popup_edge_bonus
|
|
2890
|
+
match_type = f"text='{text}'"
|
|
2891
|
+
|
|
2892
|
+
# ===== 策略2:content-desc 包含关闭关键词 =====
|
|
2893
|
+
elif any(kw in content_desc.lower() for kw in close_desc_keywords):
|
|
2894
|
+
score = 12.0 + popup_edge_bonus
|
|
2895
|
+
match_type = f"desc='{content_desc}'"
|
|
2896
|
+
|
|
2897
|
+
# ===== 策略3:clickable 的小尺寸元素(优先于非 clickable)=====
|
|
2898
|
+
elif clickable:
|
|
2899
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
2900
|
+
max_size = max(120, int(screen_width * 0.15))
|
|
2901
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2902
|
+
# clickable 元素基础分更高
|
|
2903
|
+
base_score = 8.0
|
|
2904
|
+
# 浮动关闭按钮给予最高分
|
|
2905
|
+
if is_floating_close:
|
|
2906
|
+
base_score = 12.0
|
|
2907
|
+
match_type = "floating_close"
|
|
2908
|
+
elif 'Image' in class_name:
|
|
2909
|
+
score = base_score + 2.0
|
|
2910
|
+
match_type = "clickable_image"
|
|
2911
|
+
else:
|
|
2912
|
+
match_type = "clickable"
|
|
2913
|
+
score = base_score + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2914
|
+
|
|
2915
|
+
# ===== 策略4:ImageView/ImageButton 类型的小元素(非 clickable)=====
|
|
2916
|
+
elif 'Image' in class_name:
|
|
2917
|
+
min_size = max(15, int(screen_width * 0.02))
|
|
2918
|
+
max_size = max(120, int(screen_width * 0.12))
|
|
2919
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2920
|
+
score = 5.0 + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2921
|
+
match_type = "ImageView"
|
|
2922
|
+
|
|
2923
|
+
# XML 顺序加分(后出现的元素在上层,更可能是弹窗内的元素)
|
|
2924
|
+
if score > 0:
|
|
2925
|
+
xml_order_bonus = idx / len(all_elements) * 2.0 # 最多加 2 分
|
|
2926
|
+
score += xml_order_bonus
|
|
2927
|
+
|
|
2928
|
+
close_candidates.append({
|
|
2929
|
+
'bounds': bounds_str,
|
|
2930
|
+
'center_x': center_x,
|
|
2931
|
+
'center_y': center_y,
|
|
2932
|
+
'width': width,
|
|
2933
|
+
'height': height,
|
|
2934
|
+
'score': score,
|
|
2935
|
+
'position': position,
|
|
2936
|
+
'match_type': match_type,
|
|
2937
|
+
'text': text,
|
|
2938
|
+
'content_desc': content_desc,
|
|
2939
|
+
'x_percent': round(rel_x * 100, 1),
|
|
2940
|
+
'y_percent': round(rel_y * 100, 1),
|
|
2941
|
+
'in_popup': popup_detected
|
|
2942
|
+
})
|
|
2943
|
+
|
|
2944
|
+
except ET.ParseError:
|
|
2945
|
+
pass
|
|
3858
2946
|
|
|
3859
|
-
|
|
2947
|
+
if not close_candidates:
|
|
2948
|
+
if popup_detected and popup_bounds:
|
|
2949
|
+
return {"success": False, "fallback": "vision", "popup": True}
|
|
2950
|
+
return {"success": True, "popup": False}
|
|
3860
2951
|
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
return {"success": False, "reason": f"模板匹配失败: {e}", "method": "template"}
|
|
3865
|
-
|
|
3866
|
-
def _verify_popup_closed(self) -> bool:
|
|
3867
|
-
"""验证弹窗是否已关闭(简单版本)
|
|
3868
|
-
|
|
3869
|
-
Returns:
|
|
3870
|
-
True 如果弹窗已关闭,False 如果弹窗仍在
|
|
3871
|
-
"""
|
|
3872
|
-
try:
|
|
3873
|
-
time.sleep(0.5)
|
|
3874
|
-
elements = self.list_elements()
|
|
3875
|
-
page_analysis = self._detect_page_content(elements)
|
|
3876
|
-
return not page_analysis.get("has_popup")
|
|
3877
|
-
except Exception:
|
|
3878
|
-
return False
|
|
3879
|
-
|
|
3880
|
-
def _verify_popup_closed_with_fingerprint(self, fingerprint_before: Dict = None) -> bool:
|
|
3881
|
-
"""验证弹窗是否已关闭(使用页面指纹对比)
|
|
3882
|
-
|
|
3883
|
-
Args:
|
|
3884
|
-
fingerprint_before: 操作前的页面指纹
|
|
2952
|
+
# 按得分排序,取最可能的
|
|
2953
|
+
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2954
|
+
best = close_candidates[0]
|
|
3885
2955
|
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
"""
|
|
3889
|
-
try:
|
|
2956
|
+
# 点击
|
|
2957
|
+
self.client.u2.click(best['center_x'], best['center_y'])
|
|
3890
2958
|
time.sleep(0.5)
|
|
3891
2959
|
|
|
3892
|
-
#
|
|
3893
|
-
|
|
2960
|
+
# 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
|
|
2961
|
+
app_check = self._check_app_switched()
|
|
2962
|
+
return_result = None
|
|
2963
|
+
|
|
2964
|
+
if app_check['switched']:
|
|
2965
|
+
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
2966
|
+
return_result = self._return_to_target_app()
|
|
3894
2967
|
|
|
3895
|
-
#
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
2968
|
+
# 记录操作
|
|
2969
|
+
self._record_click('percent', f"{best['x_percent']}%,{best['y_percent']}%",
|
|
2970
|
+
best['x_percent'], best['y_percent'],
|
|
2971
|
+
element_desc=f"关闭按钮({best['position']})")
|
|
3899
2972
|
|
|
3900
|
-
#
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
# (即使检测到还有弹窗特征,也可能是新弹窗或检测误判)
|
|
3907
|
-
if page_changed:
|
|
3908
|
-
return True
|
|
2973
|
+
# Token 优化:精简返回值
|
|
2974
|
+
result = {"success": True, "clicked": True}
|
|
2975
|
+
if app_check['switched']:
|
|
2976
|
+
result["switched"] = True
|
|
2977
|
+
if return_result:
|
|
2978
|
+
result["returned"] = return_result['success']
|
|
3909
2979
|
|
|
3910
|
-
return
|
|
2980
|
+
return result
|
|
3911
2981
|
|
|
3912
|
-
except Exception:
|
|
3913
|
-
return False
|
|
2982
|
+
except Exception as e:
|
|
2983
|
+
return {"success": False, "msg": str(e)}
|
|
3914
2984
|
|
|
3915
2985
|
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
3916
2986
|
"""根据相对坐标获取位置名称"""
|
|
@@ -3954,12 +3024,9 @@ class BasicMobileToolsLite:
|
|
|
3954
3024
|
else: # 中间区域
|
|
3955
3025
|
return 0.5
|
|
3956
3026
|
|
|
3957
|
-
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) ->
|
|
3027
|
+
def _detect_popup_with_confidence(self, root, screen_width: int, screen_height: int) -> tuple:
|
|
3958
3028
|
"""严格的弹窗检测 - 使用置信度评分,避免误识别普通页面
|
|
3959
3029
|
|
|
3960
|
-
使用全局常量 POPUP_CLASS_KEYWORDS、POPUP_ID_KEYWORDS、AD_POPUP_KEYWORDS、
|
|
3961
|
-
PAGE_CONTENT_KEYWORDS、CLOSE_BUTTON_KEYWORDS
|
|
3962
|
-
|
|
3963
3030
|
真正的弹窗特征:
|
|
3964
3031
|
1. class 名称包含 Dialog/Popup/Alert/Modal/BottomSheet(强特征)
|
|
3965
3032
|
2. resource-id 包含 dialog/popup/alert/modal(强特征)
|
|
@@ -3969,8 +3036,10 @@ class BasicMobileToolsLite:
|
|
|
3969
3036
|
|
|
3970
3037
|
Returns:
|
|
3971
3038
|
(popup_bounds, confidence) 或 (None, 0)
|
|
3972
|
-
confidence >=
|
|
3039
|
+
confidence >= 0.6 才认为是弹窗
|
|
3973
3040
|
"""
|
|
3041
|
+
import re
|
|
3042
|
+
|
|
3974
3043
|
screen_area = screen_width * screen_height
|
|
3975
3044
|
|
|
3976
3045
|
# 收集所有元素信息
|
|
@@ -3992,15 +3061,6 @@ class BasicMobileToolsLite:
|
|
|
3992
3061
|
class_name = elem.attrib.get('class', '')
|
|
3993
3062
|
resource_id = elem.attrib.get('resource-id', '')
|
|
3994
3063
|
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
3995
|
-
text = elem.attrib.get('text', '')
|
|
3996
|
-
|
|
3997
|
-
# 检查是否是关闭/确认按钮 - 使用全局常量
|
|
3998
|
-
is_close_button = (
|
|
3999
|
-
any(kw in resource_id.lower() for kw in CLOSE_BUTTON_KEYWORDS) or
|
|
4000
|
-
any(kw in resource_id.lower() for kw in CONFIRM_BUTTON_KEYWORDS) or
|
|
4001
|
-
text in CLOSE_BUTTON_TEXTS or
|
|
4002
|
-
text in CONFIRM_BUTTON_TEXTS
|
|
4003
|
-
)
|
|
4004
3064
|
|
|
4005
3065
|
all_elements.append({
|
|
4006
3066
|
'idx': idx,
|
|
@@ -4014,13 +3074,15 @@ class BasicMobileToolsLite:
|
|
|
4014
3074
|
'clickable': clickable,
|
|
4015
3075
|
'center_x': (x1 + x2) // 2,
|
|
4016
3076
|
'center_y': (y1 + y2) // 2,
|
|
4017
|
-
'is_close_button': is_close_button,
|
|
4018
|
-
'text': text,
|
|
4019
3077
|
})
|
|
4020
3078
|
|
|
4021
3079
|
if not all_elements:
|
|
4022
3080
|
return None, 0
|
|
4023
3081
|
|
|
3082
|
+
# 弹窗检测关键词
|
|
3083
|
+
dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
|
|
3084
|
+
dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
|
|
3085
|
+
|
|
4024
3086
|
popup_candidates = []
|
|
4025
3087
|
has_mask_layer = False
|
|
4026
3088
|
mask_idx = -1
|
|
@@ -4050,88 +3112,29 @@ class BasicMobileToolsLite:
|
|
|
4050
3112
|
if y1 < 50:
|
|
4051
3113
|
continue
|
|
4052
3114
|
|
|
4053
|
-
# 【非弹窗特征】如果元素包含底部导航栏(底部tab),则不是弹窗
|
|
4054
|
-
if y2 > screen_height * 0.85:
|
|
4055
|
-
if 'tab' in resource_id.lower() or 'Tab' in class_name or 'navigation' in resource_id.lower():
|
|
4056
|
-
continue
|
|
4057
|
-
|
|
4058
|
-
# 【非弹窗特征】如果元素包含顶部搜索栏,则不是弹窗
|
|
4059
|
-
if y1 < screen_height * 0.15:
|
|
4060
|
-
if 'search' in resource_id.lower() or 'Search' in class_name:
|
|
4061
|
-
continue
|
|
4062
|
-
|
|
4063
|
-
# 使用全局常量检查是否有强弹窗特征
|
|
4064
|
-
has_strong_popup_feature = (
|
|
4065
|
-
any(kw in class_name for kw in POPUP_CLASS_KEYWORDS) or
|
|
4066
|
-
any(kw in resource_id.lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4067
|
-
any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS)
|
|
4068
|
-
)
|
|
4069
|
-
|
|
4070
|
-
# 检查是否有子元素是关闭按钮(作为弹窗特征)
|
|
4071
|
-
has_close_button_child = False
|
|
4072
|
-
elem_bounds = elem['bounds']
|
|
4073
|
-
for other_elem in all_elements:
|
|
4074
|
-
if other_elem['idx'] == elem['idx']:
|
|
4075
|
-
continue
|
|
4076
|
-
if other_elem['is_close_button']:
|
|
4077
|
-
ox1, oy1, ox2, oy2 = other_elem['bounds']
|
|
4078
|
-
ex1, ey1, ex2, ey2 = elem_bounds
|
|
4079
|
-
if ex1 <= ox1 and ey1 <= oy1 and ex2 >= ox2 and ey2 >= oy2:
|
|
4080
|
-
has_close_button_child = True
|
|
4081
|
-
break
|
|
4082
|
-
|
|
4083
|
-
# 【非弹窗特征】使用全局常量检查页面内容特征
|
|
4084
|
-
if any(kw in resource_id.lower() or kw in class_name.lower() for kw in PAGE_CONTENT_KEYWORDS):
|
|
4085
|
-
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
4086
|
-
continue
|
|
4087
|
-
|
|
4088
|
-
# 【非弹窗特征】面积过大且无强特征
|
|
4089
|
-
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
4090
|
-
continue
|
|
4091
|
-
|
|
4092
|
-
if area_ratio > 0.7:
|
|
4093
|
-
if not has_strong_popup_feature:
|
|
4094
|
-
continue
|
|
4095
|
-
|
|
4096
3115
|
confidence = 0.0
|
|
4097
3116
|
|
|
4098
|
-
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
4099
|
-
if any(kw in class_name for kw in
|
|
3117
|
+
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
3118
|
+
if any(kw in class_name for kw in dialog_class_keywords):
|
|
4100
3119
|
confidence += 0.5
|
|
4101
3120
|
|
|
4102
|
-
# 【强特征】resource-id 包含弹窗关键词 (+0.4)
|
|
4103
|
-
if any(kw in resource_id.lower() for kw in
|
|
4104
|
-
confidence += 0.4
|
|
4105
|
-
|
|
4106
|
-
# 【强特征】resource-id 包含广告弹窗关键词 (+0.4) - 使用全局常量
|
|
4107
|
-
if any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS):
|
|
3121
|
+
# 【强特征】resource-id 包含弹窗关键词 (+0.4)
|
|
3122
|
+
if any(kw in resource_id.lower() for kw in dialog_id_keywords):
|
|
4108
3123
|
confidence += 0.4
|
|
4109
3124
|
|
|
4110
|
-
# 【强特征】包含关闭按钮作为子元素 (+0.3)
|
|
4111
|
-
if has_close_button_child:
|
|
4112
|
-
confidence += 0.3
|
|
4113
|
-
|
|
4114
3125
|
# 【中等特征】居中显示 (+0.2)
|
|
4115
3126
|
center_x = elem['center_x']
|
|
4116
3127
|
center_y = elem['center_y']
|
|
4117
3128
|
is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
|
|
4118
3129
|
is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
|
|
4119
|
-
|
|
4120
|
-
has_strong_feature = (
|
|
4121
|
-
any(kw in class_name for kw in POPUP_CLASS_KEYWORDS) or
|
|
4122
|
-
any(kw in resource_id.lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4123
|
-
any(kw in resource_id.lower() for kw in AD_POPUP_KEYWORDS) or
|
|
4124
|
-
has_close_button_child
|
|
4125
|
-
)
|
|
4126
|
-
|
|
4127
3130
|
if is_centered_x and is_centered_y:
|
|
4128
|
-
confidence += 0.2
|
|
3131
|
+
confidence += 0.2
|
|
4129
3132
|
elif is_centered_x:
|
|
4130
|
-
confidence += 0.1
|
|
3133
|
+
confidence += 0.1
|
|
4131
3134
|
|
|
4132
3135
|
# 【中等特征】非全屏但有一定大小 (+0.15)
|
|
4133
3136
|
if 0.15 < area_ratio < 0.75:
|
|
4134
|
-
confidence += 0.15
|
|
3137
|
+
confidence += 0.15
|
|
4135
3138
|
|
|
4136
3139
|
# 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
|
|
4137
3140
|
if elem['idx'] > len(all_elements) * 0.5:
|
|
@@ -4158,17 +3161,8 @@ class BasicMobileToolsLite:
|
|
|
4158
3161
|
popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
|
|
4159
3162
|
best = popup_candidates[0]
|
|
4160
3163
|
|
|
4161
|
-
#
|
|
4162
|
-
|
|
4163
|
-
any(kw in best['class'] for kw in POPUP_CLASS_KEYWORDS) or
|
|
4164
|
-
any(kw in best['resource_id'].lower() for kw in POPUP_ID_KEYWORDS) or
|
|
4165
|
-
any(kw in best['resource_id'].lower() for kw in AD_POPUP_KEYWORDS)
|
|
4166
|
-
)
|
|
4167
|
-
|
|
4168
|
-
# 有强特征时阈值 0.7,否则 0.85
|
|
4169
|
-
threshold = 0.7 if has_strong_feature else 0.85
|
|
4170
|
-
|
|
4171
|
-
if best['confidence'] >= threshold:
|
|
3164
|
+
# 只有置信度 >= 0.6 才返回弹窗
|
|
3165
|
+
if best['confidence'] >= 0.6:
|
|
4172
3166
|
return best['bounds'], best['confidence']
|
|
4173
3167
|
|
|
4174
3168
|
return None, best['confidence']
|
|
@@ -4407,6 +3401,15 @@ class BasicMobileToolsLite:
|
|
|
4407
3401
|
|
|
4408
3402
|
# 生成脚本
|
|
4409
3403
|
safe_name = re.sub(r'[^\w\s-]', '', test_name).strip().replace(' ', '_')
|
|
3404
|
+
# 确保 safe_name 不为空,否则使用默认名称
|
|
3405
|
+
if not safe_name:
|
|
3406
|
+
safe_name = 'generated_case'
|
|
3407
|
+
|
|
3408
|
+
# 提前处理文件名,确保文档字符串中的文件名正确
|
|
3409
|
+
if not filename.endswith('.py'):
|
|
3410
|
+
filename = f"{filename}.py"
|
|
3411
|
+
if not filename.startswith('test_'):
|
|
3412
|
+
filename = f"test_{filename}"
|
|
4410
3413
|
|
|
4411
3414
|
script_lines = [
|
|
4412
3415
|
"#!/usr/bin/env python3",
|
|
@@ -4421,8 +3424,8 @@ class BasicMobileToolsLite:
|
|
|
4421
3424
|
"3. 百分比定位 - 跨分辨率兼容(坐标自动转换)",
|
|
4422
3425
|
"",
|
|
4423
3426
|
"运行方式:",
|
|
4424
|
-
" pytest {filename} -v # 使用 pytest 运行",
|
|
4425
|
-
" python {filename} # 直接运行",
|
|
3427
|
+
f" pytest {filename} -v # 使用 pytest 运行",
|
|
3428
|
+
f" python {filename} # 直接运行",
|
|
4426
3429
|
f'"""',
|
|
4427
3430
|
"import time",
|
|
4428
3431
|
"import pytest",
|
|
@@ -4683,6 +3686,12 @@ class BasicMobileToolsLite:
|
|
|
4683
3686
|
script_lines.append(f" d.press('{key}')")
|
|
4684
3687
|
script_lines.append(" time.sleep(0.5)")
|
|
4685
3688
|
script_lines.append(" ")
|
|
3689
|
+
|
|
3690
|
+
elif action == 'wait':
|
|
3691
|
+
seconds = op.get('seconds', 1)
|
|
3692
|
+
script_lines.append(f" # 步骤{step_num}: 等待 {seconds} 秒")
|
|
3693
|
+
script_lines.append(f" time.sleep({seconds})")
|
|
3694
|
+
script_lines.append(" ")
|
|
4686
3695
|
|
|
4687
3696
|
script_lines.extend([
|
|
4688
3697
|
" print('✅ 测试完成')",
|
|
@@ -4706,12 +3715,6 @@ class BasicMobileToolsLite:
|
|
|
4706
3715
|
output_dir = Path("tests")
|
|
4707
3716
|
output_dir.mkdir(exist_ok=True)
|
|
4708
3717
|
|
|
4709
|
-
# 确保文件名符合 pytest 规范(以 test_ 开头)
|
|
4710
|
-
if not filename.endswith('.py'):
|
|
4711
|
-
filename = f"{filename}.py"
|
|
4712
|
-
if not filename.startswith('test_'):
|
|
4713
|
-
filename = f"test_{filename}"
|
|
4714
|
-
|
|
4715
3718
|
file_path = output_dir / filename
|
|
4716
3719
|
file_path.write_text(script, encoding='utf-8')
|
|
4717
3720
|
|
|
@@ -4853,229 +3856,235 @@ class BasicMobileToolsLite:
|
|
|
4853
3856
|
except Exception as e:
|
|
4854
3857
|
return {"success": False, "error": f"删除模板失败: {e}"}
|
|
4855
3858
|
|
|
4856
|
-
def
|
|
4857
|
-
"""
|
|
3859
|
+
def close_ad_popup(self, auto_learn: bool = True) -> Dict:
|
|
3860
|
+
"""智能关闭广告弹窗(专用于广告场景)
|
|
4858
3861
|
|
|
4859
|
-
|
|
4860
|
-
1.
|
|
4861
|
-
2.
|
|
4862
|
-
3.
|
|
3862
|
+
按优先级尝试:
|
|
3863
|
+
1. 控件树查找关闭按钮(最可靠)
|
|
3864
|
+
2. 模板匹配(需要积累模板库)
|
|
3865
|
+
3. 返回视觉信息供 AI 分析(如果前两步失败)
|
|
4863
3866
|
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
4867
|
-
- 如果通过模板匹配成功关闭,且 auto_learn=True,会自动学习新模板
|
|
3867
|
+
自动学习:
|
|
3868
|
+
- 点击成功后,检查这个 X 是否已在模板库
|
|
3869
|
+
- 如果是新样式,自动裁剪并添加到模板库
|
|
4868
3870
|
|
|
4869
3871
|
Args:
|
|
4870
|
-
auto_learn:
|
|
4871
|
-
confidence_threshold: 弹窗检测置信度阈值,默认使用 DEFAULT_POPUP_CONFIDENCE_THRESHOLD
|
|
3872
|
+
auto_learn: 是否自动学习新模板(点击成功后检查并保存)
|
|
4872
3873
|
|
|
4873
3874
|
Returns:
|
|
4874
|
-
|
|
3875
|
+
结果字典
|
|
4875
3876
|
"""
|
|
4876
|
-
|
|
4877
|
-
|
|
3877
|
+
import time
|
|
3878
|
+
import re
|
|
3879
|
+
|
|
3880
|
+
result = {
|
|
3881
|
+
"success": False,
|
|
3882
|
+
"method": None,
|
|
3883
|
+
"message": "",
|
|
3884
|
+
"learned_template": None
|
|
3885
|
+
}
|
|
3886
|
+
|
|
3887
|
+
if self._is_ios():
|
|
3888
|
+
return {"success": False, "error": "iOS 暂不支持此功能"}
|
|
4878
3889
|
|
|
4879
3890
|
try:
|
|
4880
|
-
|
|
4881
|
-
if self._is_ios():
|
|
4882
|
-
return self._close_popup_ios()
|
|
4883
|
-
|
|
4884
|
-
tried_methods = []
|
|
3891
|
+
import xml.etree.ElementTree as ET
|
|
4885
3892
|
|
|
4886
|
-
#
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
try:
|
|
4890
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
4891
|
-
except Exception:
|
|
4892
|
-
pass
|
|
3893
|
+
# ========== 第0步:先检测是否有弹窗 ==========
|
|
3894
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
3895
|
+
root = ET.fromstring(xml_string)
|
|
4893
3896
|
|
|
4894
|
-
|
|
4895
|
-
|
|
3897
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3898
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
4896
3899
|
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
elements=elements,
|
|
4900
|
-
xml_string=xml_string,
|
|
4901
|
-
confidence_threshold=confidence_threshold,
|
|
4902
|
-
page_fingerprint_before=page_fingerprint_before
|
|
3900
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3901
|
+
root, screen_width, screen_height
|
|
4903
3902
|
)
|
|
4904
|
-
tried_methods.append("elements")
|
|
4905
|
-
if result.get("success") and result.get("verified", False):
|
|
4906
|
-
return result
|
|
4907
|
-
# 如果明确检测到没有弹窗,直接返回,不继续后续阶段
|
|
4908
|
-
if result.get("reason") == "未检测到弹窗":
|
|
4909
|
-
return {
|
|
4910
|
-
"success": False,
|
|
4911
|
-
"message": "✅ 未检测到弹窗,无需关闭",
|
|
4912
|
-
"method": "elements",
|
|
4913
|
-
"tried_methods": tried_methods
|
|
4914
|
-
}
|
|
4915
3903
|
|
|
4916
|
-
#
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
tip += f"\n💡 候选关闭按钮(按优先级):"
|
|
4925
|
-
for i, c in enumerate(candidates[:3], 1):
|
|
4926
|
-
tip += f"\n {i}. {c.get('reason', '')} at ({c.get('x_percent', 0)}%, {c.get('y_percent', 0)}%)"
|
|
4927
|
-
|
|
4928
|
-
return {
|
|
4929
|
-
**result,
|
|
4930
|
-
"tried_methods": tried_methods,
|
|
4931
|
-
"tip": tip
|
|
4932
|
-
}
|
|
3904
|
+
# 如果没有检测到弹窗,直接返回"无弹窗"
|
|
3905
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
3906
|
+
result["success"] = True
|
|
3907
|
+
result["method"] = None
|
|
3908
|
+
result["message"] = "ℹ️ 当前页面未检测到弹窗,无需关闭"
|
|
3909
|
+
result["popup_detected"] = False
|
|
3910
|
+
result["popup_confidence"] = popup_confidence
|
|
3911
|
+
return result
|
|
4933
3912
|
|
|
4934
|
-
#
|
|
4935
|
-
pre_screenshot_path = None
|
|
4936
|
-
if auto_learn:
|
|
4937
|
-
try:
|
|
4938
|
-
pre_screenshot_result = self.take_screenshot(description="关闭前(用于自动学习)", compress=False)
|
|
4939
|
-
pre_screenshot_path = pre_screenshot_result.get("screenshot_path")
|
|
4940
|
-
except Exception:
|
|
4941
|
-
pass
|
|
3913
|
+
# ========== 第1步:控件树查找关闭按钮 ==========
|
|
4942
3914
|
|
|
4943
|
-
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
if auto_learn and result.get("method") == "template" and pre_screenshot_path:
|
|
4947
|
-
clicked = result.get("clicked", {})
|
|
4948
|
-
if clicked and "center" in clicked:
|
|
4949
|
-
x, y = clicked["center"]
|
|
4950
|
-
bounds = (x - 30, y - 30, x + 30, y + 30)
|
|
4951
|
-
try:
|
|
4952
|
-
learn_result = self._auto_learn_template(pre_screenshot_path, bounds)
|
|
4953
|
-
if learn_result:
|
|
4954
|
-
result["learned_template"] = learn_result
|
|
4955
|
-
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
4956
|
-
except Exception:
|
|
4957
|
-
pass
|
|
4958
|
-
|
|
4959
|
-
return result
|
|
3915
|
+
# 关闭按钮的常见特征
|
|
3916
|
+
close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
|
|
3917
|
+
close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
|
|
4960
3918
|
|
|
4961
|
-
|
|
4962
|
-
return {
|
|
4963
|
-
"success": False,
|
|
4964
|
-
"message": "❌ 所有关闭弹窗方法都失败",
|
|
4965
|
-
"tried_methods": tried_methods,
|
|
4966
|
-
"suggestion": "请手动查看截图,找到关闭按钮位置,然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
4967
|
-
}
|
|
3919
|
+
close_candidates = []
|
|
4968
3920
|
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
4972
|
-
|
|
4973
|
-
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
|
|
4980
|
-
|
|
3921
|
+
for elem in root.iter():
|
|
3922
|
+
text = elem.attrib.get('text', '').strip()
|
|
3923
|
+
content_desc = elem.attrib.get('content-desc', '').strip()
|
|
3924
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
3925
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
3926
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
3927
|
+
|
|
3928
|
+
if not bounds_str:
|
|
3929
|
+
continue
|
|
3930
|
+
|
|
3931
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
3932
|
+
if not match:
|
|
3933
|
+
continue
|
|
3934
|
+
|
|
3935
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
3936
|
+
width = x2 - x1
|
|
3937
|
+
height = y2 - y1
|
|
3938
|
+
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
|
|
3939
|
+
|
|
3940
|
+
score = 0
|
|
3941
|
+
reason = ""
|
|
3942
|
+
|
|
3943
|
+
# 文本匹配
|
|
3944
|
+
for kw in close_keywords:
|
|
3945
|
+
if kw in text:
|
|
3946
|
+
score += 10
|
|
3947
|
+
reason = f"文本含'{kw}'"
|
|
3948
|
+
break
|
|
3949
|
+
|
|
3950
|
+
# content-desc 匹配
|
|
3951
|
+
for kw in close_content_desc:
|
|
3952
|
+
if kw.lower() in content_desc.lower():
|
|
3953
|
+
score += 8
|
|
3954
|
+
reason = f"描述含'{kw}'"
|
|
3955
|
+
break
|
|
3956
|
+
|
|
3957
|
+
# 小尺寸可点击元素(可能是 X 按钮)
|
|
3958
|
+
if clickable and 30 < width < 200 and 30 < height < 200:
|
|
3959
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3960
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
3961
|
+
|
|
3962
|
+
# 在屏幕右半边上半部分,很可能是 X
|
|
3963
|
+
if cx > screen_width * 0.6 and cy < screen_height * 0.5:
|
|
3964
|
+
score += 5
|
|
3965
|
+
reason = reason or "右上角小按钮"
|
|
3966
|
+
# 在屏幕上半部分的小按钮,也可能是 X
|
|
3967
|
+
elif cy < screen_height * 0.4:
|
|
3968
|
+
score += 2
|
|
3969
|
+
reason = reason or "上部小按钮"
|
|
3970
|
+
|
|
3971
|
+
# 只要是可点击的小按钮都考虑(即使没有文本)
|
|
3972
|
+
if score > 0 or (clickable and 30 < width < 150 and 30 < height < 150):
|
|
3973
|
+
if not reason and clickable:
|
|
3974
|
+
reason = "可点击小按钮"
|
|
3975
|
+
score = max(score, 1) # 确保有分数
|
|
3976
|
+
close_candidates.append({
|
|
3977
|
+
'score': score,
|
|
3978
|
+
'reason': reason,
|
|
3979
|
+
'bounds': (x1, y1, x2, y2),
|
|
3980
|
+
'center': (cx, cy),
|
|
3981
|
+
'resource_id': resource_id,
|
|
3982
|
+
'text': text
|
|
3983
|
+
})
|
|
4981
3984
|
|
|
4982
|
-
|
|
4983
|
-
|
|
3985
|
+
# 按分数排序
|
|
3986
|
+
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
4984
3987
|
|
|
4985
|
-
|
|
3988
|
+
if close_candidates:
|
|
3989
|
+
best = close_candidates[0]
|
|
3990
|
+
cx, cy = best['center']
|
|
3991
|
+
bounds = best['bounds']
|
|
3992
|
+
|
|
3993
|
+
# 点击(click_at_coords 内部已包含应用状态检查和自动返回)
|
|
3994
|
+
click_result = self.click_at_coords(cx, cy)
|
|
3995
|
+
time.sleep(0.5)
|
|
3996
|
+
|
|
3997
|
+
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
3998
|
+
app_check = self._check_app_switched()
|
|
3999
|
+
return_result = None
|
|
4000
|
+
|
|
4001
|
+
if app_check['switched']:
|
|
4002
|
+
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
4003
|
+
return_result = self._return_to_target_app()
|
|
4004
|
+
|
|
4005
|
+
result["success"] = True
|
|
4006
|
+
result["method"] = "控件树"
|
|
4007
|
+
msg = f"✅ 通过控件树找到关闭按钮并点击\n" \
|
|
4008
|
+
f" 位置: ({cx}, {cy})\n" \
|
|
4009
|
+
f" 原因: {best['reason']}"
|
|
4010
|
+
|
|
4011
|
+
if app_check['switched']:
|
|
4012
|
+
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
4013
|
+
if return_result:
|
|
4014
|
+
if return_result['success']:
|
|
4015
|
+
msg += f"\n{return_result['message']}"
|
|
4016
|
+
else:
|
|
4017
|
+
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
4018
|
+
|
|
4019
|
+
result["message"] = msg
|
|
4020
|
+
result["app_check"] = app_check
|
|
4021
|
+
result["return_to_app"] = return_result
|
|
4022
|
+
result["tip"] = "💡 建议调用 mobile_screenshot_with_som 确认弹窗是否已关闭"
|
|
4023
|
+
|
|
4024
|
+
return result
|
|
4986
4025
|
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4026
|
+
# ========== 第2步:模板匹配(自动执行,不需要 AI 介入)==========
|
|
4027
|
+
screenshot_path = None
|
|
4028
|
+
try:
|
|
4029
|
+
from .template_matcher import TemplateMatcher
|
|
4030
|
+
|
|
4031
|
+
# 截图用于模板匹配
|
|
4032
|
+
screenshot_result = self.take_screenshot(description="模板匹配", compress=False)
|
|
4033
|
+
screenshot_path = screenshot_result.get("screenshot_path")
|
|
4034
|
+
|
|
4035
|
+
if screenshot_path:
|
|
4036
|
+
matcher = TemplateMatcher()
|
|
4037
|
+
match_result = matcher.find_close_buttons(screenshot_path, threshold=0.75)
|
|
4992
4038
|
|
|
4993
|
-
#
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4039
|
+
# 直接使用最佳匹配(已按置信度排序)
|
|
4040
|
+
if match_result.get("success") and match_result.get("best_match"):
|
|
4041
|
+
best = match_result["best_match"]
|
|
4042
|
+
x_pct = best["percent"]["x"]
|
|
4043
|
+
y_pct = best["percent"]["y"]
|
|
4044
|
+
|
|
4045
|
+
# 点击
|
|
4046
|
+
click_result = self.click_by_percent(x_pct, y_pct)
|
|
4047
|
+
time.sleep(0.5)
|
|
4048
|
+
|
|
4049
|
+
app_check = self._check_app_switched()
|
|
4050
|
+
return_result = None
|
|
4051
|
+
|
|
4052
|
+
if app_check['switched']:
|
|
4053
|
+
return_result = self._return_to_target_app()
|
|
4054
|
+
|
|
4055
|
+
result["success"] = True
|
|
4056
|
+
result["method"] = "模板匹配"
|
|
4057
|
+
msg = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
|
|
4058
|
+
f" 模板: {best.get('template', 'unknown')}\n" \
|
|
4059
|
+
f" 置信度: {best.get('confidence', 'N/A')}%\n" \
|
|
4060
|
+
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
4061
|
+
|
|
4062
|
+
if app_check['switched']:
|
|
4063
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
4064
|
+
if return_result:
|
|
4065
|
+
msg += f"\n{return_result['message']}"
|
|
4066
|
+
|
|
4067
|
+
result["message"] = msg
|
|
4068
|
+
result["app_check"] = app_check
|
|
4069
|
+
result["return_to_app"] = return_result
|
|
4070
|
+
return result
|
|
4997
4071
|
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
"clicked": {
|
|
5003
|
-
"center": (close_button['center_x'], close_button['center_y']),
|
|
5004
|
-
"percent": (close_button['x_percent'], close_button['y_percent'])
|
|
5005
|
-
},
|
|
5006
|
-
"verified": verified
|
|
5007
|
-
}
|
|
5008
|
-
|
|
5009
|
-
return {"success": False, "reason": "未找到关闭按钮", "method": "ios_elements"}
|
|
4072
|
+
except ImportError:
|
|
4073
|
+
pass # OpenCV 未安装,跳过模板匹配
|
|
4074
|
+
except Exception:
|
|
4075
|
+
pass # 模板匹配失败,继续下一步
|
|
5010
4076
|
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
|
|
5016
|
-
|
|
5017
|
-
Args:
|
|
5018
|
-
elements: 元素列表
|
|
4077
|
+
# ========== 第3步:控件树和模板匹配都失败,提示 AI 使用视觉识别 ==========
|
|
4078
|
+
result["success"] = False
|
|
4079
|
+
result["fallback"] = "vision"
|
|
4080
|
+
result["method"] = None
|
|
4081
|
+
result["popup_detected"] = True
|
|
4082
|
+
result["message"] = "⚠️ 控件树和模板匹配都未找到关闭按钮,请调用 mobile_screenshot_with_som 截图后用 click_by_som 点击"
|
|
5019
4083
|
|
|
5020
|
-
|
|
5021
|
-
页面指纹信息
|
|
5022
|
-
"""
|
|
5023
|
-
# 提取关键特征
|
|
5024
|
-
resource_ids = set()
|
|
5025
|
-
texts = set()
|
|
5026
|
-
clickable_count = 0
|
|
5027
|
-
|
|
5028
|
-
for elem in elements:
|
|
5029
|
-
rid = elem.get("resource_id", "")
|
|
5030
|
-
if rid:
|
|
5031
|
-
resource_ids.add(rid)
|
|
5032
|
-
text = elem.get("text", "")
|
|
5033
|
-
if text and len(text) < 50: # 只记录短文本
|
|
5034
|
-
texts.add(text)
|
|
5035
|
-
if elem.get("clickable"):
|
|
5036
|
-
clickable_count += 1
|
|
5037
|
-
|
|
5038
|
-
return {
|
|
5039
|
-
"element_count": len(elements),
|
|
5040
|
-
"clickable_count": clickable_count,
|
|
5041
|
-
"resource_ids": resource_ids,
|
|
5042
|
-
"texts": texts,
|
|
5043
|
-
"resource_id_hash": hash(frozenset(resource_ids)) if resource_ids else 0,
|
|
5044
|
-
}
|
|
5045
|
-
|
|
5046
|
-
def _compare_page_fingerprint(self, before: Dict, after: Dict) -> bool:
|
|
5047
|
-
"""比较页面指纹,判断页面是否发生变化
|
|
5048
|
-
|
|
5049
|
-
Args:
|
|
5050
|
-
before: 操作前的页面指纹
|
|
5051
|
-
after: 操作后的页面指纹
|
|
4084
|
+
return result
|
|
5052
4085
|
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
"""
|
|
5056
|
-
if not before or not after:
|
|
5057
|
-
return False
|
|
5058
|
-
|
|
5059
|
-
# 元素数量变化
|
|
5060
|
-
count_diff = abs(after["element_count"] - before["element_count"])
|
|
5061
|
-
count_change_ratio = count_diff / max(before["element_count"], 1)
|
|
5062
|
-
|
|
5063
|
-
# resource_id 集合变化
|
|
5064
|
-
before_ids = before.get("resource_ids", set())
|
|
5065
|
-
after_ids = after.get("resource_ids", set())
|
|
5066
|
-
id_diff = len(before_ids.symmetric_difference(after_ids))
|
|
5067
|
-
id_change_ratio = id_diff / max(len(before_ids), 1)
|
|
5068
|
-
|
|
5069
|
-
# 如果元素数量或 resource_id 发生明显变化,认为页面已变化
|
|
5070
|
-
if count_change_ratio > 0.1 or id_change_ratio > 0.1:
|
|
5071
|
-
return True
|
|
5072
|
-
|
|
5073
|
-
# 如果 hash 值不同,也认为有变化
|
|
5074
|
-
if before.get("resource_id_hash") != after.get("resource_id_hash"):
|
|
5075
|
-
return True
|
|
5076
|
-
|
|
5077
|
-
return False
|
|
5078
|
-
|
|
4086
|
+
except Exception as e:
|
|
4087
|
+
return {"success": False, "error": f"关闭弹窗失败: {e}"}
|
|
5079
4088
|
|
|
5080
4089
|
def _detect_popup_region(self, root) -> tuple:
|
|
5081
4090
|
"""从控件树中检测弹窗区域
|