mobile-mcp-ai 2.6.8__py3-none-any.whl → 2.6.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/config.py +32 -0
- mobile_mcp/core/basic_tools_lite.py +385 -739
- mobile_mcp/core/tool_selection_helper.py +168 -0
- mobile_mcp/mcp_tools/mcp_server.py +336 -391
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/METADATA +19 -12
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/RECORD +10 -9
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/WHEEL +0 -0
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/entry_points.txt +0 -0
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/licenses/LICENSE +0 -0
- {mobile_mcp_ai-2.6.8.dist-info → mobile_mcp_ai-2.6.12.dist-info}/top_level.txt +0 -0
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
- 核心功能精简
|
|
9
9
|
- 保留 pytest 脚本生成
|
|
10
10
|
- 支持操作历史记录
|
|
11
|
+
- Token 优化模式(省钱)
|
|
11
12
|
"""
|
|
12
13
|
|
|
13
14
|
import asyncio
|
|
@@ -17,6 +18,19 @@ from pathlib import Path
|
|
|
17
18
|
from typing import Dict, List, Optional
|
|
18
19
|
from datetime import datetime
|
|
19
20
|
|
|
21
|
+
# Token 优化配置(只精简格式,不限制数量,确保准确度)
|
|
22
|
+
try:
|
|
23
|
+
from mobile_mcp.config import Config
|
|
24
|
+
TOKEN_OPTIMIZATION = Config.TOKEN_OPTIMIZATION_ENABLED
|
|
25
|
+
MAX_ELEMENTS = Config.MAX_ELEMENTS_RETURN
|
|
26
|
+
MAX_SOM_ELEMENTS = Config.MAX_SOM_ELEMENTS_RETURN
|
|
27
|
+
COMPACT_RESPONSE = Config.COMPACT_RESPONSE
|
|
28
|
+
except ImportError:
|
|
29
|
+
TOKEN_OPTIMIZATION = True
|
|
30
|
+
MAX_ELEMENTS = 0 # 0 = 不限制
|
|
31
|
+
MAX_SOM_ELEMENTS = 0 # 0 = 不限制
|
|
32
|
+
COMPACT_RESPONSE = True
|
|
33
|
+
|
|
20
34
|
|
|
21
35
|
class BasicMobileToolsLite:
|
|
22
36
|
"""精简版移动端工具"""
|
|
@@ -335,7 +349,7 @@ class BasicMobileToolsLite:
|
|
|
335
349
|
size = ios_client.wda.window_size()
|
|
336
350
|
screen_width, screen_height = size[0], size[1]
|
|
337
351
|
else:
|
|
338
|
-
return {"success": False, "
|
|
352
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
339
353
|
else:
|
|
340
354
|
self.client.u2.screenshot(str(temp_path))
|
|
341
355
|
info = self.client.u2.info
|
|
@@ -386,22 +400,14 @@ class BasicMobileToolsLite:
|
|
|
386
400
|
|
|
387
401
|
cropped_size = final_path.stat().st_size
|
|
388
402
|
|
|
403
|
+
# 返回结果
|
|
389
404
|
return {
|
|
390
405
|
"success": True,
|
|
391
406
|
"screenshot_path": str(final_path),
|
|
392
|
-
"screen_width": screen_width,
|
|
393
|
-
"screen_height": screen_height,
|
|
394
407
|
"image_width": img.width,
|
|
395
408
|
"image_height": img.height,
|
|
396
409
|
"crop_offset_x": crop_offset_x,
|
|
397
|
-
"crop_offset_y": crop_offset_y
|
|
398
|
-
"file_size": f"{cropped_size/1024:.1f}KB",
|
|
399
|
-
"message": f"🔍 局部截图已保存: {final_path}\n"
|
|
400
|
-
f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
|
|
401
|
-
f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
|
|
402
|
-
f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
|
|
403
|
-
f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
|
|
404
|
-
f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
|
|
410
|
+
"crop_offset_y": crop_offset_y
|
|
405
411
|
}
|
|
406
412
|
|
|
407
413
|
# ========== 情况2:全屏压缩截图 ==========
|
|
@@ -454,24 +460,14 @@ class BasicMobileToolsLite:
|
|
|
454
460
|
compressed_size = final_path.stat().st_size
|
|
455
461
|
saved_percent = (1 - compressed_size / original_size) * 100
|
|
456
462
|
|
|
463
|
+
# 返回结果
|
|
457
464
|
return {
|
|
458
465
|
"success": True,
|
|
459
466
|
"screenshot_path": str(final_path),
|
|
460
|
-
"
|
|
461
|
-
"
|
|
462
|
-
"original_img_width": original_img_width,
|
|
463
|
-
"original_img_height": original_img_height
|
|
464
|
-
"image_width": image_width, # 压缩后宽度(AI 看到的)
|
|
465
|
-
"image_height": image_height, # 压缩后高度(AI 看到的)
|
|
466
|
-
"original_size": f"{original_size/1024:.1f}KB",
|
|
467
|
-
"compressed_size": f"{compressed_size/1024:.1f}KB",
|
|
468
|
-
"saved_percent": f"{saved_percent:.0f}%",
|
|
469
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
470
|
-
f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
|
|
471
|
-
f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
|
|
472
|
-
f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
|
|
473
|
-
f" image_width={image_width}, image_height={image_height},\n"
|
|
474
|
-
f" original_img_width={original_img_width}, original_img_height={original_img_height}"
|
|
467
|
+
"image_width": image_width,
|
|
468
|
+
"image_height": image_height,
|
|
469
|
+
"original_img_width": original_img_width,
|
|
470
|
+
"original_img_height": original_img_height
|
|
475
471
|
}
|
|
476
472
|
|
|
477
473
|
# ========== 情况3:全屏不压缩截图 ==========
|
|
@@ -485,21 +481,12 @@ class BasicMobileToolsLite:
|
|
|
485
481
|
final_path = self.screenshot_dir / filename
|
|
486
482
|
temp_path.rename(final_path)
|
|
487
483
|
|
|
488
|
-
#
|
|
484
|
+
# 返回结果(不压缩时尺寸相同)
|
|
489
485
|
return {
|
|
490
486
|
"success": True,
|
|
491
487
|
"screenshot_path": str(final_path),
|
|
492
|
-
"
|
|
493
|
-
"
|
|
494
|
-
"original_img_width": img.width, # 截图实际尺寸
|
|
495
|
-
"original_img_height": img.height,
|
|
496
|
-
"image_width": img.width, # 未压缩,和原图一样
|
|
497
|
-
"image_height": img.height,
|
|
498
|
-
"file_size": f"{original_size/1024:.1f}KB",
|
|
499
|
-
"message": f"📸 截图已保存: {final_path}\n"
|
|
500
|
-
f"📐 截图尺寸: {img.width}x{img.height}\n"
|
|
501
|
-
f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
|
|
502
|
-
f"💡 未压缩,坐标可直接使用"
|
|
488
|
+
"image_width": img.width,
|
|
489
|
+
"image_height": img.height
|
|
503
490
|
}
|
|
504
491
|
except ImportError:
|
|
505
492
|
# 如果没有 PIL,回退到原始方式(不压缩)
|
|
@@ -539,7 +526,7 @@ class BasicMobileToolsLite:
|
|
|
539
526
|
size = ios_client.wda.window_size()
|
|
540
527
|
screen_width, screen_height = size[0], size[1]
|
|
541
528
|
else:
|
|
542
|
-
return {"success": False, "
|
|
529
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
543
530
|
else:
|
|
544
531
|
self.client.u2.screenshot(str(temp_path))
|
|
545
532
|
info = self.client.u2.info
|
|
@@ -653,26 +640,16 @@ class BasicMobileToolsLite:
|
|
|
653
640
|
result = {
|
|
654
641
|
"success": True,
|
|
655
642
|
"screenshot_path": str(final_path),
|
|
656
|
-
"screen_width": screen_width,
|
|
657
|
-
"screen_height": screen_height,
|
|
658
643
|
"image_width": img_width,
|
|
659
644
|
"image_height": img_height,
|
|
660
|
-
"grid_size": grid_size
|
|
661
|
-
"message": f"📸 网格截图已保存: {final_path}\n"
|
|
662
|
-
f"📐 尺寸: {img_width}x{img_height}\n"
|
|
663
|
-
f"📏 网格间距: {grid_size}px"
|
|
645
|
+
"grid_size": grid_size
|
|
664
646
|
}
|
|
665
647
|
|
|
666
648
|
if popup_info:
|
|
667
|
-
result["
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
|
|
672
|
-
for pos in close_positions:
|
|
673
|
-
result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
|
|
674
|
-
else:
|
|
675
|
-
result["popup_detected"] = False
|
|
649
|
+
result["popup"] = popup_info["bounds"]
|
|
650
|
+
# 只返回前3个最可能的关闭按钮位置
|
|
651
|
+
if close_positions:
|
|
652
|
+
result["close_hints"] = [(p['x'], p['y']) for p in close_positions[:3]]
|
|
676
653
|
|
|
677
654
|
return result
|
|
678
655
|
|
|
@@ -709,7 +686,7 @@ class BasicMobileToolsLite:
|
|
|
709
686
|
size = ios_client.wda.window_size()
|
|
710
687
|
screen_width, screen_height = size[0], size[1]
|
|
711
688
|
else:
|
|
712
|
-
return {"success": False, "
|
|
689
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
713
690
|
else:
|
|
714
691
|
self.client.u2.screenshot(str(temp_path))
|
|
715
692
|
info = self.client.u2.info
|
|
@@ -869,38 +846,15 @@ class BasicMobileToolsLite:
|
|
|
869
846
|
img.save(str(final_path), "JPEG", quality=85)
|
|
870
847
|
temp_path.unlink()
|
|
871
848
|
|
|
872
|
-
#
|
|
873
|
-
elements_text = "\n".join([
|
|
874
|
-
f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
|
|
875
|
-
for e in som_elements[:15] # 只显示前15个
|
|
876
|
-
])
|
|
877
|
-
if len(som_elements) > 15:
|
|
878
|
-
elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
|
|
879
|
-
|
|
880
|
-
# 构建弹窗提示文字
|
|
881
|
-
hints_text = ""
|
|
882
|
-
if popup_bounds:
|
|
883
|
-
hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
|
|
884
|
-
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
885
|
-
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
886
|
-
|
|
849
|
+
# 返回结果(Token 优化:不返回 elements 列表,已存储在 self._som_elements)
|
|
887
850
|
return {
|
|
888
851
|
"success": True,
|
|
889
852
|
"screenshot_path": str(final_path),
|
|
890
853
|
"screen_width": screen_width,
|
|
891
854
|
"screen_height": screen_height,
|
|
892
|
-
"image_width": img_width,
|
|
893
|
-
"image_height": img_height,
|
|
894
855
|
"element_count": len(som_elements),
|
|
895
|
-
"elements": som_elements,
|
|
896
856
|
"popup_detected": popup_bounds is not None,
|
|
897
|
-
"
|
|
898
|
-
"message": f"📸 SoM 截图已保存: {final_path}\n"
|
|
899
|
-
f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
900
|
-
f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
901
|
-
f"💡 使用方法:\n"
|
|
902
|
-
f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
903
|
-
f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
857
|
+
"hint": "查看截图上的编号,用 click_by_som(编号) 点击"
|
|
904
858
|
}
|
|
905
859
|
|
|
906
860
|
except ImportError:
|
|
@@ -981,7 +935,6 @@ class BasicMobileToolsLite:
|
|
|
981
935
|
|
|
982
936
|
return {
|
|
983
937
|
"success": True,
|
|
984
|
-
"message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
|
|
985
938
|
"clicked": {
|
|
986
939
|
"index": index,
|
|
987
940
|
"desc": target['desc'],
|
|
@@ -1015,7 +968,7 @@ class BasicMobileToolsLite:
|
|
|
1015
968
|
size = ios_client.wda.window_size()
|
|
1016
969
|
width, height = size[0], size[1]
|
|
1017
970
|
else:
|
|
1018
|
-
return {"success": False, "
|
|
971
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1019
972
|
else:
|
|
1020
973
|
self.client.u2.screenshot(str(screenshot_path))
|
|
1021
974
|
info = self.client.u2.info
|
|
@@ -1093,7 +1046,7 @@ class BasicMobileToolsLite:
|
|
|
1093
1046
|
size = ios_client.wda.window_size()
|
|
1094
1047
|
screen_width, screen_height = size[0], size[1]
|
|
1095
1048
|
else:
|
|
1096
|
-
return {"success": False, "
|
|
1049
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1097
1050
|
else:
|
|
1098
1051
|
info = self.client.u2.info
|
|
1099
1052
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1208,14 +1161,14 @@ class BasicMobileToolsLite:
|
|
|
1208
1161
|
size = ios_client.wda.window_size()
|
|
1209
1162
|
width, height = size[0], size[1]
|
|
1210
1163
|
else:
|
|
1211
|
-
return {"success": False, "
|
|
1164
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1212
1165
|
else:
|
|
1213
1166
|
info = self.client.u2.info
|
|
1214
1167
|
width = info.get('displayWidth', 0)
|
|
1215
1168
|
height = info.get('displayHeight', 0)
|
|
1216
1169
|
|
|
1217
1170
|
if width == 0 or height == 0:
|
|
1218
|
-
return {"success": False, "
|
|
1171
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1219
1172
|
|
|
1220
1173
|
# 第2步:百分比转像素坐标
|
|
1221
1174
|
# 公式:像素 = 屏幕尺寸 × (百分比 / 100)
|
|
@@ -1236,15 +1189,13 @@ class BasicMobileToolsLite:
|
|
|
1236
1189
|
|
|
1237
1190
|
return {
|
|
1238
1191
|
"success": True,
|
|
1239
|
-
"message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
|
|
1240
|
-
"screen_size": {"width": width, "height": height},
|
|
1241
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1242
1192
|
"pixel": {"x": x, "y": y}
|
|
1243
1193
|
}
|
|
1244
1194
|
except Exception as e:
|
|
1245
1195
|
return {"success": False, "message": f"❌ 百分比点击失败: {e}"}
|
|
1246
1196
|
|
|
1247
|
-
def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None
|
|
1197
|
+
def click_by_text(self, text: str, timeout: float = 3.0, position: Optional[str] = None,
|
|
1198
|
+
verify: Optional[str] = None) -> Dict:
|
|
1248
1199
|
"""通过文本点击 - 先查 XML 树,再精准匹配
|
|
1249
1200
|
|
|
1250
1201
|
Args:
|
|
@@ -1253,6 +1204,7 @@ class BasicMobileToolsLite:
|
|
|
1253
1204
|
position: 位置信息,当有多个相同文案时使用。支持:
|
|
1254
1205
|
- 垂直方向: "top"/"upper"/"上", "bottom"/"lower"/"下", "middle"/"center"/"中"
|
|
1255
1206
|
- 水平方向: "left"/"左", "right"/"右", "center"/"中"
|
|
1207
|
+
verify: 可选,点击后验证的文本。如果指定,会检查该文本是否出现在页面上
|
|
1256
1208
|
"""
|
|
1257
1209
|
try:
|
|
1258
1210
|
if self._is_ios():
|
|
@@ -1264,10 +1216,17 @@ class BasicMobileToolsLite:
|
|
|
1264
1216
|
if elem.exists:
|
|
1265
1217
|
elem.click()
|
|
1266
1218
|
time.sleep(0.3)
|
|
1267
|
-
# 使用标准记录格式
|
|
1268
1219
|
self._record_click('text', text, element_desc=text, locator_attr='text')
|
|
1269
|
-
|
|
1270
|
-
|
|
1220
|
+
# 验证逻辑
|
|
1221
|
+
if verify:
|
|
1222
|
+
return self._verify_after_click(verify, ios=True)
|
|
1223
|
+
# 返回页面文本摘要,方便确认页面变化
|
|
1224
|
+
page_texts = self._get_page_texts(10)
|
|
1225
|
+
return {"success": True, "page_texts": page_texts}
|
|
1226
|
+
# 控件树找不到,提示用视觉识别
|
|
1227
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1228
|
+
else:
|
|
1229
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1271
1230
|
else:
|
|
1272
1231
|
# 获取屏幕尺寸用于计算百分比
|
|
1273
1232
|
screen_width, screen_height = self.client.u2.window_size()
|
|
@@ -1288,17 +1247,20 @@ class BasicMobileToolsLite:
|
|
|
1288
1247
|
x_pct = round(cx / screen_width * 100, 1)
|
|
1289
1248
|
y_pct = round(cy / screen_height * 100, 1)
|
|
1290
1249
|
|
|
1291
|
-
#
|
|
1250
|
+
# 如果有位置参数,直接使用坐标点击
|
|
1292
1251
|
if position and bounds:
|
|
1293
1252
|
x = (bounds[0] + bounds[2]) // 2
|
|
1294
1253
|
y = (bounds[1] + bounds[3]) // 2
|
|
1295
1254
|
self.client.u2.click(x, y)
|
|
1296
1255
|
time.sleep(0.3)
|
|
1297
|
-
position_info = f" ({position})" if position else ""
|
|
1298
|
-
# 虽然用坐标点击,但记录时仍使用文本定位(脚本更稳定)
|
|
1299
1256
|
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1300
|
-
element_desc=f"{text}{
|
|
1301
|
-
|
|
1257
|
+
element_desc=f"{text}({position})", locator_attr=attr_type)
|
|
1258
|
+
# 验证逻辑
|
|
1259
|
+
if verify:
|
|
1260
|
+
return self._verify_after_click(verify)
|
|
1261
|
+
# 返回页面文本摘要
|
|
1262
|
+
page_texts = self._get_page_texts(10)
|
|
1263
|
+
return {"success": True, "page_texts": page_texts}
|
|
1302
1264
|
|
|
1303
1265
|
# 没有位置参数时,使用选择器定位
|
|
1304
1266
|
if attr_type == 'text':
|
|
@@ -1315,27 +1277,74 @@ class BasicMobileToolsLite:
|
|
|
1315
1277
|
if elem and elem.exists(timeout=1):
|
|
1316
1278
|
elem.click()
|
|
1317
1279
|
time.sleep(0.3)
|
|
1318
|
-
position_info = f" ({position})" if position else ""
|
|
1319
|
-
# 使用标准记录格式:文本定位
|
|
1320
1280
|
self._record_click('text', attr_value, x_pct, y_pct,
|
|
1321
1281
|
element_desc=text, locator_attr=attr_type)
|
|
1322
|
-
|
|
1282
|
+
# 验证逻辑
|
|
1283
|
+
if verify:
|
|
1284
|
+
return self._verify_after_click(verify)
|
|
1285
|
+
# 返回页面文本摘要
|
|
1286
|
+
page_texts = self._get_page_texts(10)
|
|
1287
|
+
return {"success": True, "page_texts": page_texts}
|
|
1323
1288
|
|
|
1324
|
-
#
|
|
1289
|
+
# 选择器失败,用坐标兜底
|
|
1325
1290
|
if bounds:
|
|
1326
1291
|
x = (bounds[0] + bounds[2]) // 2
|
|
1327
1292
|
y = (bounds[1] + bounds[3]) // 2
|
|
1328
1293
|
self.client.u2.click(x, y)
|
|
1329
1294
|
time.sleep(0.3)
|
|
1330
|
-
position_info = f" ({position})" if position else ""
|
|
1331
|
-
# 选择器失败,用百分比作为兜底
|
|
1332
1295
|
self._record_click('percent', f"{x_pct}%,{y_pct}%", x_pct, y_pct,
|
|
1333
|
-
element_desc=
|
|
1334
|
-
|
|
1296
|
+
element_desc=text)
|
|
1297
|
+
# 验证逻辑
|
|
1298
|
+
if verify:
|
|
1299
|
+
return self._verify_after_click(verify)
|
|
1300
|
+
# 返回页面文本摘要
|
|
1301
|
+
page_texts = self._get_page_texts(10)
|
|
1302
|
+
return {"success": True, "page_texts": page_texts}
|
|
1335
1303
|
|
|
1336
|
-
|
|
1304
|
+
# 控件树找不到,提示用视觉识别
|
|
1305
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到'{text}',用截图点击"}
|
|
1337
1306
|
except Exception as e:
|
|
1338
|
-
return {"success": False, "
|
|
1307
|
+
return {"success": False, "msg": str(e)}
|
|
1308
|
+
|
|
1309
|
+
def _verify_after_click(self, verify_text: str, ios: bool = False, timeout: float = 2.0) -> Dict:
|
|
1310
|
+
"""点击后验证期望文本是否出现
|
|
1311
|
+
|
|
1312
|
+
Args:
|
|
1313
|
+
verify_text: 期望出现的文本
|
|
1314
|
+
ios: 是否是 iOS 设备
|
|
1315
|
+
timeout: 验证超时时间
|
|
1316
|
+
|
|
1317
|
+
Returns:
|
|
1318
|
+
{"success": True, "verified": True/False, "hint": "..."}
|
|
1319
|
+
"""
|
|
1320
|
+
time.sleep(0.5) # 等待页面更新
|
|
1321
|
+
|
|
1322
|
+
try:
|
|
1323
|
+
if ios:
|
|
1324
|
+
ios_client = self._get_ios_client()
|
|
1325
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1326
|
+
exists = ios_client.wda(name=verify_text).exists or \
|
|
1327
|
+
ios_client.wda(label=verify_text).exists
|
|
1328
|
+
else:
|
|
1329
|
+
exists = False
|
|
1330
|
+
else:
|
|
1331
|
+
# Android: 检查文本或包含文本
|
|
1332
|
+
exists = self.client.u2(text=verify_text).exists(timeout=timeout) or \
|
|
1333
|
+
self.client.u2(textContains=verify_text).exists(timeout=0.5) or \
|
|
1334
|
+
self.client.u2(description=verify_text).exists(timeout=0.5)
|
|
1335
|
+
|
|
1336
|
+
if exists:
|
|
1337
|
+
return {"success": True, "verified": True}
|
|
1338
|
+
else:
|
|
1339
|
+
# 验证失败,提示可以截图确认
|
|
1340
|
+
return {
|
|
1341
|
+
"success": True, # 点击本身成功
|
|
1342
|
+
"verified": False,
|
|
1343
|
+
"expect": verify_text,
|
|
1344
|
+
"hint": "验证失败,可截图确认"
|
|
1345
|
+
}
|
|
1346
|
+
except Exception as e:
|
|
1347
|
+
return {"success": True, "verified": False, "hint": f"验证异常: {e}"}
|
|
1339
1348
|
|
|
1340
1349
|
def _find_element_in_tree(self, text: str, position: Optional[str] = None) -> Optional[Dict]:
|
|
1341
1350
|
"""在 XML 树中查找包含指定文本的元素,优先返回可点击的元素
|
|
@@ -1474,15 +1483,8 @@ class BasicMobileToolsLite:
|
|
|
1474
1483
|
return None
|
|
1475
1484
|
|
|
1476
1485
|
def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
|
|
1477
|
-
"""通过 resource-id
|
|
1478
|
-
|
|
1479
|
-
Args:
|
|
1480
|
-
resource_id: 元素的 resource-id
|
|
1481
|
-
index: 第几个元素(从 0 开始),默认 0 表示第一个
|
|
1482
|
-
"""
|
|
1486
|
+
"""通过 resource-id 点击"""
|
|
1483
1487
|
try:
|
|
1484
|
-
index_desc = f"[{index}]" if index > 0 else ""
|
|
1485
|
-
|
|
1486
1488
|
if self._is_ios():
|
|
1487
1489
|
ios_client = self._get_ios_client()
|
|
1488
1490
|
if ios_client and hasattr(ios_client, 'wda'):
|
|
@@ -1490,33 +1492,31 @@ class BasicMobileToolsLite:
|
|
|
1490
1492
|
if not elem.exists:
|
|
1491
1493
|
elem = ios_client.wda(name=resource_id)
|
|
1492
1494
|
if elem.exists:
|
|
1493
|
-
# 获取所有匹配的元素
|
|
1494
1495
|
elements = elem.find_elements()
|
|
1495
1496
|
if index < len(elements):
|
|
1496
1497
|
elements[index].click()
|
|
1497
1498
|
time.sleep(0.3)
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}"}
|
|
1499
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1500
|
+
return {"success": True}
|
|
1501
1501
|
else:
|
|
1502
|
-
return {"success": False, "
|
|
1503
|
-
return {"success": False, "
|
|
1502
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{len(elements)}个)"}
|
|
1503
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1504
|
+
else:
|
|
1505
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1504
1506
|
else:
|
|
1505
1507
|
elem = self.client.u2(resourceId=resource_id)
|
|
1506
1508
|
if elem.exists(timeout=0.5):
|
|
1507
|
-
# 获取匹配元素数量
|
|
1508
1509
|
count = elem.count
|
|
1509
1510
|
if index < count:
|
|
1510
1511
|
elem[index].click()
|
|
1511
1512
|
time.sleep(0.3)
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}" + (f" (共 {count} 个)" if count > 1 else "")}
|
|
1513
|
+
self._record_click('id', resource_id, element_desc=resource_id)
|
|
1514
|
+
return {"success": True}
|
|
1515
1515
|
else:
|
|
1516
|
-
return {"success": False, "
|
|
1517
|
-
return {"success": False, "
|
|
1516
|
+
return {"success": False, "msg": f"索引{index}超出范围(共{count}个)"}
|
|
1517
|
+
return {"success": False, "fallback": "vision", "msg": f"未找到ID'{resource_id}'"}
|
|
1518
1518
|
except Exception as e:
|
|
1519
|
-
return {"success": False, "
|
|
1519
|
+
return {"success": False, "msg": str(e)}
|
|
1520
1520
|
|
|
1521
1521
|
# ==================== 长按操作 ====================
|
|
1522
1522
|
|
|
@@ -1550,7 +1550,7 @@ class BasicMobileToolsLite:
|
|
|
1550
1550
|
size = ios_client.wda.window_size()
|
|
1551
1551
|
screen_width, screen_height = size[0], size[1]
|
|
1552
1552
|
else:
|
|
1553
|
-
return {"success": False, "
|
|
1553
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1554
1554
|
else:
|
|
1555
1555
|
info = self.client.u2.info
|
|
1556
1556
|
screen_width = info.get('displayWidth', 0)
|
|
@@ -1603,23 +1603,11 @@ class BasicMobileToolsLite:
|
|
|
1603
1603
|
|
|
1604
1604
|
if converted:
|
|
1605
1605
|
if conversion_type == "crop_offset":
|
|
1606
|
-
return {
|
|
1607
|
-
"success": True,
|
|
1608
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1609
|
-
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
1610
|
-
}
|
|
1606
|
+
return {"success": True}
|
|
1611
1607
|
else:
|
|
1612
|
-
return {
|
|
1613
|
-
"success": True,
|
|
1614
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1615
|
-
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
1616
|
-
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
1617
|
-
}
|
|
1608
|
+
return {"success": True}
|
|
1618
1609
|
else:
|
|
1619
|
-
return {
|
|
1620
|
-
"success": True,
|
|
1621
|
-
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
|
|
1622
|
-
}
|
|
1610
|
+
return {"success": True}
|
|
1623
1611
|
except Exception as e:
|
|
1624
1612
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1625
1613
|
|
|
@@ -1648,14 +1636,14 @@ class BasicMobileToolsLite:
|
|
|
1648
1636
|
size = ios_client.wda.window_size()
|
|
1649
1637
|
width, height = size[0], size[1]
|
|
1650
1638
|
else:
|
|
1651
|
-
return {"success": False, "
|
|
1639
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
1652
1640
|
else:
|
|
1653
1641
|
info = self.client.u2.info
|
|
1654
1642
|
width = info.get('displayWidth', 0)
|
|
1655
1643
|
height = info.get('displayHeight', 0)
|
|
1656
1644
|
|
|
1657
1645
|
if width == 0 or height == 0:
|
|
1658
|
-
return {"success": False, "
|
|
1646
|
+
return {"success": False, "msg": "无法获取屏幕尺寸"}
|
|
1659
1647
|
|
|
1660
1648
|
# 第2步:百分比转像素坐标
|
|
1661
1649
|
x = int(width * x_percent / 100)
|
|
@@ -1677,13 +1665,7 @@ class BasicMobileToolsLite:
|
|
|
1677
1665
|
self._record_long_press('percent', f"{x_percent}%,{y_percent}%", duration,
|
|
1678
1666
|
x_percent, y_percent, element_desc=f"百分比({x_percent}%,{y_percent}%)")
|
|
1679
1667
|
|
|
1680
|
-
return {
|
|
1681
|
-
"success": True,
|
|
1682
|
-
"message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
|
|
1683
|
-
"screen_size": {"width": width, "height": height},
|
|
1684
|
-
"percent": {"x": x_percent, "y": y_percent},
|
|
1685
|
-
"pixel": {"x": x, "y": y},
|
|
1686
|
-
"duration": duration
|
|
1668
|
+
return {"success": True
|
|
1687
1669
|
}
|
|
1688
1670
|
except Exception as e:
|
|
1689
1671
|
return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
|
|
@@ -1713,8 +1695,8 @@ class BasicMobileToolsLite:
|
|
|
1713
1695
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1714
1696
|
time.sleep(0.3)
|
|
1715
1697
|
self._record_long_press('text', text, duration, element_desc=text, locator_attr='text')
|
|
1716
|
-
return {"success": True
|
|
1717
|
-
return {"success": False, "
|
|
1698
|
+
return {"success": True}
|
|
1699
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1718
1700
|
else:
|
|
1719
1701
|
# 获取屏幕尺寸用于计算百分比
|
|
1720
1702
|
screen_width, screen_height = self.client.u2.window_size()
|
|
@@ -1752,7 +1734,7 @@ class BasicMobileToolsLite:
|
|
|
1752
1734
|
time.sleep(0.3)
|
|
1753
1735
|
self._record_long_press('text', attr_value, duration, x_pct, y_pct,
|
|
1754
1736
|
element_desc=text, locator_attr=attr_type)
|
|
1755
|
-
return {"success": True
|
|
1737
|
+
return {"success": True}
|
|
1756
1738
|
|
|
1757
1739
|
# 如果选择器失败,用坐标兜底
|
|
1758
1740
|
if bounds:
|
|
@@ -1762,9 +1744,9 @@ class BasicMobileToolsLite:
|
|
|
1762
1744
|
time.sleep(0.3)
|
|
1763
1745
|
self._record_long_press('percent', f"{x_pct}%,{y_pct}%", duration, x_pct, y_pct,
|
|
1764
1746
|
element_desc=text)
|
|
1765
|
-
return {"success": True
|
|
1747
|
+
return {"success": True}
|
|
1766
1748
|
|
|
1767
|
-
return {"success": False, "
|
|
1749
|
+
return {"success": False, "msg": f"未找到'{text}'"}
|
|
1768
1750
|
except Exception as e:
|
|
1769
1751
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1770
1752
|
|
|
@@ -1792,8 +1774,8 @@ class BasicMobileToolsLite:
|
|
|
1792
1774
|
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1793
1775
|
time.sleep(0.3)
|
|
1794
1776
|
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1795
|
-
return {"success": True
|
|
1796
|
-
return {"success": False, "
|
|
1777
|
+
return {"success": True}
|
|
1778
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1797
1779
|
else:
|
|
1798
1780
|
elem = self.client.u2(resourceId=resource_id)
|
|
1799
1781
|
if elem.exists(timeout=0.5):
|
|
@@ -1801,7 +1783,7 @@ class BasicMobileToolsLite:
|
|
|
1801
1783
|
time.sleep(0.3)
|
|
1802
1784
|
self._record_long_press('id', resource_id, duration, element_desc=resource_id)
|
|
1803
1785
|
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
1804
|
-
return {"success": False, "
|
|
1786
|
+
return {"success": False, "msg": f"未找到'{resource_id}'"}
|
|
1805
1787
|
except Exception as e:
|
|
1806
1788
|
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1807
1789
|
|
|
@@ -2058,15 +2040,8 @@ class BasicMobileToolsLite:
|
|
|
2058
2040
|
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
2059
2041
|
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
2060
2042
|
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
x=x,
|
|
2064
|
-
y=y,
|
|
2065
|
-
x_percent=x_percent,
|
|
2066
|
-
y_percent=y_percent,
|
|
2067
|
-
ref=f"coords_{x}_{y}",
|
|
2068
|
-
text=text
|
|
2069
|
-
)
|
|
2043
|
+
# 使用标准记录格式
|
|
2044
|
+
self._record_input(text, 'percent', f"{x_percent}%,{y_percent}%", x_percent, y_percent)
|
|
2070
2045
|
|
|
2071
2046
|
# 🎯 关键步骤:检查应用是否跳转,如果跳转则自动返回目标应用
|
|
2072
2047
|
app_check = self._check_app_switched()
|
|
@@ -2096,16 +2071,13 @@ class BasicMobileToolsLite:
|
|
|
2096
2071
|
|
|
2097
2072
|
# ==================== 导航操作 ====================
|
|
2098
2073
|
|
|
2099
|
-
async def swipe(self, direction: str, y: Optional[int] = None, y_percent: Optional[float] = None
|
|
2100
|
-
distance: Optional[int] = None, distance_percent: Optional[float] = None) -> Dict:
|
|
2074
|
+
async def swipe(self, direction: str, y: Optional[int] = None, y_percent: Optional[float] = None) -> Dict:
|
|
2101
2075
|
"""滑动屏幕
|
|
2102
2076
|
|
|
2103
2077
|
Args:
|
|
2104
2078
|
direction: 滑动方向 (up/down/left/right)
|
|
2105
2079
|
y: 左右滑动时指定的高度坐标(像素)
|
|
2106
2080
|
y_percent: 左右滑动时指定的高度百分比 (0-100)
|
|
2107
|
-
distance: 横向滑动时指定的滑动距离(像素),仅用于 left/right
|
|
2108
|
-
distance_percent: 横向滑动时指定的滑动距离百分比 (0-100),仅用于 left/right
|
|
2109
2081
|
"""
|
|
2110
2082
|
try:
|
|
2111
2083
|
if self._is_ios():
|
|
@@ -2114,7 +2086,7 @@ class BasicMobileToolsLite:
|
|
|
2114
2086
|
size = ios_client.wda.window_size()
|
|
2115
2087
|
width, height = size[0], size[1]
|
|
2116
2088
|
else:
|
|
2117
|
-
return {"success": False, "
|
|
2089
|
+
return {"success": False, "msg": "iOS未初始化"}
|
|
2118
2090
|
else:
|
|
2119
2091
|
width, height = self.client.u2.window_size()
|
|
2120
2092
|
|
|
@@ -2132,53 +2104,20 @@ class BasicMobileToolsLite:
|
|
|
2132
2104
|
swipe_y = y
|
|
2133
2105
|
else:
|
|
2134
2106
|
swipe_y = center_y
|
|
2135
|
-
|
|
2136
|
-
# 计算横向滑动距离
|
|
2137
|
-
if distance_percent is not None:
|
|
2138
|
-
if not (0 <= distance_percent <= 100):
|
|
2139
|
-
return {"success": False, "message": f"❌ distance_percent 必须在 0-100 之间: {distance_percent}"}
|
|
2140
|
-
swipe_distance = int(width * distance_percent / 100)
|
|
2141
|
-
elif distance is not None:
|
|
2142
|
-
if distance <= 0:
|
|
2143
|
-
return {"success": False, "message": f"❌ distance 必须大于 0: {distance}"}
|
|
2144
|
-
if distance > width:
|
|
2145
|
-
return {"success": False, "message": f"❌ distance 不能超过屏幕宽度 ({width}): {distance}"}
|
|
2146
|
-
swipe_distance = distance
|
|
2147
|
-
else:
|
|
2148
|
-
# 默认滑动距离:屏幕宽度的 60%(从 0.8 到 0.2)
|
|
2149
|
-
swipe_distance = int(width * 0.6)
|
|
2150
|
-
|
|
2151
|
-
# 计算起始和结束位置
|
|
2152
|
-
if direction == 'left':
|
|
2153
|
-
# 从右向左滑动:起始点在右侧,结束点在左侧
|
|
2154
|
-
# 确保起始点不超出屏幕右边界
|
|
2155
|
-
start_x = min(center_x + swipe_distance // 2, width - 10)
|
|
2156
|
-
end_x = start_x - swipe_distance
|
|
2157
|
-
# 确保结束点不超出屏幕左边界
|
|
2158
|
-
if end_x < 10:
|
|
2159
|
-
end_x = 10
|
|
2160
|
-
start_x = min(end_x + swipe_distance, width - 10)
|
|
2161
|
-
else: # right
|
|
2162
|
-
# 从左向右滑动:起始点在左侧,结束点在右侧
|
|
2163
|
-
# 确保起始点不超出屏幕左边界
|
|
2164
|
-
start_x = max(center_x - swipe_distance // 2, 10)
|
|
2165
|
-
end_x = start_x + swipe_distance
|
|
2166
|
-
# 确保结束点不超出屏幕右边界
|
|
2167
|
-
if end_x > width - 10:
|
|
2168
|
-
end_x = width - 10
|
|
2169
|
-
start_x = max(end_x - swipe_distance, 10)
|
|
2170
|
-
|
|
2171
|
-
x1, y1, x2, y2 = start_x, swipe_y, end_x, swipe_y
|
|
2172
2107
|
else:
|
|
2173
2108
|
swipe_y = center_y
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2109
|
+
|
|
2110
|
+
swipe_map = {
|
|
2111
|
+
'up': (center_x, int(height * 0.8), center_x, int(height * 0.2)),
|
|
2112
|
+
'down': (center_x, int(height * 0.2), center_x, int(height * 0.8)),
|
|
2113
|
+
'left': (int(width * 0.8), swipe_y, int(width * 0.2), swipe_y),
|
|
2114
|
+
'right': (int(width * 0.2), swipe_y, int(width * 0.8), swipe_y),
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
if direction not in swipe_map:
|
|
2118
|
+
return {"success": False, "message": f"❌ 不支持的方向: {direction}"}
|
|
2119
|
+
|
|
2120
|
+
x1, y1, x2, y2 = swipe_map[direction]
|
|
2182
2121
|
|
|
2183
2122
|
if self._is_ios():
|
|
2184
2123
|
ios_client.wda.swipe(x1, y1, x2, y2)
|
|
@@ -2199,21 +2138,10 @@ class BasicMobileToolsLite:
|
|
|
2199
2138
|
# 构建返回消息
|
|
2200
2139
|
msg = f"✅ 滑动成功: {direction}"
|
|
2201
2140
|
if direction in ['left', 'right']:
|
|
2202
|
-
msg_parts = []
|
|
2203
2141
|
if y_percent is not None:
|
|
2204
|
-
|
|
2142
|
+
msg += f" (高度: {y_percent}% = {swipe_y}px)"
|
|
2205
2143
|
elif y is not None:
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
if distance_percent is not None:
|
|
2209
|
-
msg_parts.append(f"距离: {distance_percent}% = {swipe_distance}px")
|
|
2210
|
-
elif distance is not None:
|
|
2211
|
-
msg_parts.append(f"距离: {distance}px")
|
|
2212
|
-
else:
|
|
2213
|
-
msg_parts.append(f"距离: 默认 {swipe_distance}px")
|
|
2214
|
-
|
|
2215
|
-
if msg_parts:
|
|
2216
|
-
msg += f" ({', '.join(msg_parts)})"
|
|
2144
|
+
msg += f" (高度: {y}px)"
|
|
2217
2145
|
|
|
2218
2146
|
# 如果检测到应用跳转,添加警告和返回结果
|
|
2219
2147
|
if app_check['switched']:
|
|
@@ -2254,186 +2182,22 @@ class BasicMobileToolsLite:
|
|
|
2254
2182
|
ios_client.wda.send_keys('\n')
|
|
2255
2183
|
elif ios_key == 'home':
|
|
2256
2184
|
ios_client.wda.home()
|
|
2257
|
-
return {"success": True
|
|
2258
|
-
return {"success": False, "
|
|
2185
|
+
return {"success": True}
|
|
2186
|
+
return {"success": False, "msg": f"iOS不支持{key}"}
|
|
2259
2187
|
else:
|
|
2260
2188
|
keycode = key_map.get(key.lower())
|
|
2261
2189
|
if keycode:
|
|
2262
2190
|
self.client.u2.shell(f'input keyevent {keycode}')
|
|
2263
2191
|
self._record_key(key)
|
|
2264
|
-
return {"success": True
|
|
2265
|
-
return {"success": False, "
|
|
2192
|
+
return {"success": True}
|
|
2193
|
+
return {"success": False, "msg": f"不支持按键{key}"}
|
|
2266
2194
|
except Exception as e:
|
|
2267
2195
|
return {"success": False, "message": f"❌ 按键失败: {e}"}
|
|
2268
2196
|
|
|
2269
2197
|
def wait(self, seconds: float) -> Dict:
|
|
2270
2198
|
"""等待指定时间"""
|
|
2271
2199
|
time.sleep(seconds)
|
|
2272
|
-
return {"success": True
|
|
2273
|
-
|
|
2274
|
-
async def drag_progress_bar(self, direction: str = "right", distance_percent: float = 30.0,
|
|
2275
|
-
y_percent: Optional[float] = None, y: Optional[int] = None) -> Dict:
|
|
2276
|
-
"""智能拖动进度条
|
|
2277
|
-
|
|
2278
|
-
自动检测进度条是否可见:
|
|
2279
|
-
- 如果进度条已显示,直接拖动(无需先点击播放区域)
|
|
2280
|
-
- 如果进度条未显示,先点击播放区域显示控制栏,再拖动
|
|
2281
|
-
|
|
2282
|
-
Args:
|
|
2283
|
-
direction: 拖动方向,'left'(倒退)或 'right'(前进),默认 'right'
|
|
2284
|
-
distance_percent: 拖动距离百分比 (0-100),默认 30%
|
|
2285
|
-
y_percent: 进度条的垂直位置百分比 (0-100),如果未指定则自动检测
|
|
2286
|
-
y: 进度条的垂直位置坐标(像素),如果未指定则自动检测
|
|
2287
|
-
"""
|
|
2288
|
-
try:
|
|
2289
|
-
import xml.etree.ElementTree as ET
|
|
2290
|
-
import re
|
|
2291
|
-
|
|
2292
|
-
if self._is_ios():
|
|
2293
|
-
return {"success": False, "message": "❌ iOS 暂不支持,请使用 mobile_swipe"}
|
|
2294
|
-
|
|
2295
|
-
if direction not in ['left', 'right']:
|
|
2296
|
-
return {"success": False, "message": f"❌ 拖动方向必须是 'left' 或 'right': {direction}"}
|
|
2297
|
-
|
|
2298
|
-
screen_width, screen_height = self.client.u2.window_size()
|
|
2299
|
-
|
|
2300
|
-
# 获取 XML 查找进度条
|
|
2301
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2302
|
-
root = ET.fromstring(xml_string)
|
|
2303
|
-
|
|
2304
|
-
progress_bar_found = False
|
|
2305
|
-
progress_bar_y = None
|
|
2306
|
-
progress_bar_y_percent = None
|
|
2307
|
-
|
|
2308
|
-
# 查找进度条元素(SeekBar、ProgressBar)
|
|
2309
|
-
for elem in root.iter():
|
|
2310
|
-
class_name = elem.attrib.get('class', '')
|
|
2311
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2312
|
-
bounds_str = elem.attrib.get('bounds', '')
|
|
2313
|
-
|
|
2314
|
-
# 检查是否是进度条
|
|
2315
|
-
is_progress_bar = (
|
|
2316
|
-
'SeekBar' in class_name or
|
|
2317
|
-
'ProgressBar' in class_name or
|
|
2318
|
-
'progress' in resource_id.lower() or
|
|
2319
|
-
'seek' in resource_id.lower()
|
|
2320
|
-
)
|
|
2321
|
-
|
|
2322
|
-
if is_progress_bar and bounds_str:
|
|
2323
|
-
# 解析 bounds 获取进度条位置
|
|
2324
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2325
|
-
if match:
|
|
2326
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2327
|
-
center_y = (y1 + y2) // 2
|
|
2328
|
-
progress_bar_y = center_y
|
|
2329
|
-
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2330
|
-
progress_bar_found = True
|
|
2331
|
-
break
|
|
2332
|
-
|
|
2333
|
-
# 如果未找到进度条,尝试点击播放区域显示控制栏
|
|
2334
|
-
if not progress_bar_found:
|
|
2335
|
-
# 点击屏幕中心显示控制栏
|
|
2336
|
-
center_x, center_y = screen_width // 2, screen_height // 2
|
|
2337
|
-
self.client.u2.click(center_x, center_y)
|
|
2338
|
-
time.sleep(0.5)
|
|
2339
|
-
|
|
2340
|
-
# 再次查找进度条
|
|
2341
|
-
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2342
|
-
root = ET.fromstring(xml_string)
|
|
2343
|
-
|
|
2344
|
-
for elem in root.iter():
|
|
2345
|
-
class_name = elem.attrib.get('class', '')
|
|
2346
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2347
|
-
bounds_str = elem.attrib.get('bounds', '')
|
|
2348
|
-
|
|
2349
|
-
is_progress_bar = (
|
|
2350
|
-
'SeekBar' in class_name or
|
|
2351
|
-
'ProgressBar' in class_name or
|
|
2352
|
-
'progress' in resource_id.lower() or
|
|
2353
|
-
'seek' in resource_id.lower()
|
|
2354
|
-
)
|
|
2355
|
-
|
|
2356
|
-
if is_progress_bar and bounds_str:
|
|
2357
|
-
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2358
|
-
if match:
|
|
2359
|
-
x1, y1, x2, y2 = map(int, match.groups())
|
|
2360
|
-
center_y = (y1 + y2) // 2
|
|
2361
|
-
progress_bar_y = center_y
|
|
2362
|
-
progress_bar_y_percent = round(center_y / screen_height * 100, 1)
|
|
2363
|
-
progress_bar_found = True
|
|
2364
|
-
break
|
|
2365
|
-
|
|
2366
|
-
# 确定使用的高度位置
|
|
2367
|
-
if y_percent is not None:
|
|
2368
|
-
swipe_y = int(screen_height * y_percent / 100)
|
|
2369
|
-
used_y_percent = y_percent
|
|
2370
|
-
elif y is not None:
|
|
2371
|
-
swipe_y = y
|
|
2372
|
-
used_y_percent = round(y / screen_height * 100, 1)
|
|
2373
|
-
elif progress_bar_found:
|
|
2374
|
-
swipe_y = progress_bar_y
|
|
2375
|
-
used_y_percent = progress_bar_y_percent
|
|
2376
|
-
else:
|
|
2377
|
-
# 默认使用屏幕底部附近(进度条常见位置)
|
|
2378
|
-
swipe_y = int(screen_height * 0.91)
|
|
2379
|
-
used_y_percent = 91.0
|
|
2380
|
-
|
|
2381
|
-
# 计算滑动距离
|
|
2382
|
-
swipe_distance = int(screen_width * distance_percent / 100)
|
|
2383
|
-
|
|
2384
|
-
# 计算起始和结束位置
|
|
2385
|
-
center_x = screen_width // 2
|
|
2386
|
-
if direction == 'left':
|
|
2387
|
-
start_x = min(center_x + swipe_distance // 2, screen_width - 10)
|
|
2388
|
-
end_x = start_x - swipe_distance
|
|
2389
|
-
if end_x < 10:
|
|
2390
|
-
end_x = 10
|
|
2391
|
-
start_x = min(end_x + swipe_distance, screen_width - 10)
|
|
2392
|
-
else: # right
|
|
2393
|
-
start_x = max(center_x - swipe_distance // 2, 10)
|
|
2394
|
-
end_x = start_x + swipe_distance
|
|
2395
|
-
if end_x > screen_width - 10:
|
|
2396
|
-
end_x = screen_width - 10
|
|
2397
|
-
start_x = max(end_x - swipe_distance, 10)
|
|
2398
|
-
|
|
2399
|
-
# 执行拖动
|
|
2400
|
-
self.client.u2.swipe(start_x, swipe_y, end_x, swipe_y, duration=0.5)
|
|
2401
|
-
time.sleep(0.3)
|
|
2402
|
-
|
|
2403
|
-
# 记录操作
|
|
2404
|
-
self._record_swipe(direction)
|
|
2405
|
-
|
|
2406
|
-
# 检查应用是否跳转
|
|
2407
|
-
app_check = self._check_app_switched()
|
|
2408
|
-
return_result = None
|
|
2409
|
-
if app_check['switched']:
|
|
2410
|
-
return_result = self._return_to_target_app()
|
|
2411
|
-
|
|
2412
|
-
# 构建返回消息
|
|
2413
|
-
msg = f"✅ 进度条拖动成功: {direction} (高度: {used_y_percent}%, 距离: {distance_percent}%)"
|
|
2414
|
-
if not progress_bar_found:
|
|
2415
|
-
msg += "\n💡 已自动点击播放区域显示控制栏"
|
|
2416
|
-
else:
|
|
2417
|
-
msg += "\n💡 进度条已显示,直接拖动"
|
|
2418
|
-
|
|
2419
|
-
if app_check['switched']:
|
|
2420
|
-
msg += f"\n{app_check['message']}"
|
|
2421
|
-
if return_result and return_result.get('success'):
|
|
2422
|
-
msg += f"\n{return_result['message']}"
|
|
2423
|
-
|
|
2424
|
-
return {
|
|
2425
|
-
"success": True,
|
|
2426
|
-
"message": msg,
|
|
2427
|
-
"progress_bar_found": progress_bar_found,
|
|
2428
|
-
"y_percent": used_y_percent,
|
|
2429
|
-
"distance_percent": distance_percent,
|
|
2430
|
-
"direction": direction,
|
|
2431
|
-
"app_check": app_check,
|
|
2432
|
-
"return_to_app": return_result
|
|
2433
|
-
}
|
|
2434
|
-
|
|
2435
|
-
except Exception as e:
|
|
2436
|
-
return {"success": False, "message": f"❌ 拖动进度条失败: {e}"}
|
|
2200
|
+
return {"success": True}
|
|
2437
2201
|
|
|
2438
2202
|
# ==================== 应用管理 ====================
|
|
2439
2203
|
|
|
@@ -2462,10 +2226,7 @@ class BasicMobileToolsLite:
|
|
|
2462
2226
|
|
|
2463
2227
|
self._record_operation('launch_app', package_name=package_name)
|
|
2464
2228
|
|
|
2465
|
-
return {
|
|
2466
|
-
"success": True,
|
|
2467
|
-
"message": f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载\n📱 已设置应用状态监测"
|
|
2468
|
-
}
|
|
2229
|
+
return {"success": True}
|
|
2469
2230
|
except Exception as e:
|
|
2470
2231
|
return {"success": False, "message": f"❌ 启动失败: {e}"}
|
|
2471
2232
|
|
|
@@ -2478,9 +2239,9 @@ class BasicMobileToolsLite:
|
|
|
2478
2239
|
ios_client.wda.app_terminate(package_name)
|
|
2479
2240
|
else:
|
|
2480
2241
|
self.client.u2.app_stop(package_name)
|
|
2481
|
-
return {"success": True
|
|
2242
|
+
return {"success": True}
|
|
2482
2243
|
except Exception as e:
|
|
2483
|
-
return {"success": False, "
|
|
2244
|
+
return {"success": False, "msg": str(e)}
|
|
2484
2245
|
|
|
2485
2246
|
def list_apps(self, filter_keyword: str = "") -> Dict:
|
|
2486
2247
|
"""列出已安装应用"""
|
|
@@ -2592,6 +2353,26 @@ class BasicMobileToolsLite:
|
|
|
2592
2353
|
'_shadow', 'shadow_', '_divider', 'divider_', '_line', 'line_'
|
|
2593
2354
|
}
|
|
2594
2355
|
|
|
2356
|
+
# Token 优化:构建精简元素(只返回非空字段)
|
|
2357
|
+
def build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name):
|
|
2358
|
+
"""只返回有值的字段,节省 token"""
|
|
2359
|
+
item = {}
|
|
2360
|
+
if resource_id:
|
|
2361
|
+
# 精简 resource_id,只保留最后一段
|
|
2362
|
+
item['id'] = resource_id.split('/')[-1] if '/' in resource_id else resource_id
|
|
2363
|
+
if text:
|
|
2364
|
+
item['text'] = text
|
|
2365
|
+
if content_desc:
|
|
2366
|
+
item['desc'] = content_desc
|
|
2367
|
+
if bounds:
|
|
2368
|
+
item['bounds'] = bounds
|
|
2369
|
+
if likely_click:
|
|
2370
|
+
item['click'] = True # 启发式判断可点击
|
|
2371
|
+
# class 精简:只保留关键类型
|
|
2372
|
+
if class_name in ('EditText', 'TextInput', 'Button', 'ImageButton', 'CheckBox', 'Switch'):
|
|
2373
|
+
item['type'] = class_name
|
|
2374
|
+
return item
|
|
2375
|
+
|
|
2595
2376
|
result = []
|
|
2596
2377
|
for elem in elements:
|
|
2597
2378
|
# 获取元素属性
|
|
@@ -2611,14 +2392,11 @@ class BasicMobileToolsLite:
|
|
|
2611
2392
|
|
|
2612
2393
|
# 2. 检查是否是功能控件(直接保留)
|
|
2613
2394
|
if class_name in FUNCTIONAL_WIDGETS:
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
'clickable': clickable,
|
|
2620
|
-
'class': class_name
|
|
2621
|
-
})
|
|
2395
|
+
# 使用启发式判断可点击性(替代不准确的 clickable 属性)
|
|
2396
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2397
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2398
|
+
if item:
|
|
2399
|
+
result.append(item)
|
|
2622
2400
|
continue
|
|
2623
2401
|
|
|
2624
2402
|
# 3. 检查是否是容器控件
|
|
@@ -2631,14 +2409,10 @@ class BasicMobileToolsLite:
|
|
|
2631
2409
|
# 所有属性都是默认值,过滤掉
|
|
2632
2410
|
continue
|
|
2633
2411
|
# 有业务ID或其他有意义属性,保留
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
'bounds': bounds,
|
|
2639
|
-
'clickable': clickable,
|
|
2640
|
-
'class': class_name
|
|
2641
|
-
})
|
|
2412
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2413
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2414
|
+
if item:
|
|
2415
|
+
result.append(item)
|
|
2642
2416
|
continue
|
|
2643
2417
|
|
|
2644
2418
|
# 4. 检查是否是装饰类控件
|
|
@@ -2655,19 +2429,72 @@ class BasicMobileToolsLite:
|
|
|
2655
2429
|
continue
|
|
2656
2430
|
|
|
2657
2431
|
# 6. 其他情况:有意义的元素保留
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2432
|
+
likely_click = self._is_likely_clickable(class_name, resource_id, text, content_desc, clickable, bounds)
|
|
2433
|
+
item = build_compact_element(resource_id, text, content_desc, bounds, likely_click, class_name)
|
|
2434
|
+
if item:
|
|
2435
|
+
result.append(item)
|
|
2436
|
+
|
|
2437
|
+
# Token 优化:可选限制返回元素数量(默认不限制,确保准确度)
|
|
2438
|
+
if TOKEN_OPTIMIZATION and MAX_ELEMENTS > 0 and len(result) > MAX_ELEMENTS:
|
|
2439
|
+
# 仅在用户明确设置 MAX_ELEMENTS_RETURN 时才截断
|
|
2440
|
+
truncated = result[:MAX_ELEMENTS]
|
|
2441
|
+
truncated.append({
|
|
2442
|
+
'_truncated': True,
|
|
2443
|
+
'_total': len(result),
|
|
2444
|
+
'_shown': MAX_ELEMENTS
|
|
2665
2445
|
})
|
|
2446
|
+
return truncated
|
|
2666
2447
|
|
|
2667
2448
|
return result
|
|
2668
2449
|
except Exception as e:
|
|
2669
2450
|
return [{"error": f"获取元素失败: {e}"}]
|
|
2670
2451
|
|
|
2452
|
+
def _get_page_texts(self, max_count: int = 15) -> List[str]:
|
|
2453
|
+
"""获取页面关键文本列表(用于点击后快速确认页面变化)
|
|
2454
|
+
|
|
2455
|
+
Args:
|
|
2456
|
+
max_count: 最多返回的文本数量
|
|
2457
|
+
|
|
2458
|
+
Returns:
|
|
2459
|
+
页面上的关键文本列表(去重)
|
|
2460
|
+
"""
|
|
2461
|
+
try:
|
|
2462
|
+
if self._is_ios():
|
|
2463
|
+
ios_client = self._get_ios_client()
|
|
2464
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
2465
|
+
# iOS: 获取所有 StaticText 的文本
|
|
2466
|
+
elements = ios_client.wda(type='XCUIElementTypeStaticText').find_elements()
|
|
2467
|
+
texts = set()
|
|
2468
|
+
for elem in elements[:50]: # 限制扫描数量
|
|
2469
|
+
try:
|
|
2470
|
+
name = elem.name or elem.label
|
|
2471
|
+
if name and len(name) > 1 and len(name) < 50:
|
|
2472
|
+
texts.add(name)
|
|
2473
|
+
except:
|
|
2474
|
+
pass
|
|
2475
|
+
return list(texts)[:max_count]
|
|
2476
|
+
return []
|
|
2477
|
+
else:
|
|
2478
|
+
# Android: 快速扫描 XML 获取文本
|
|
2479
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=True)
|
|
2480
|
+
import xml.etree.ElementTree as ET
|
|
2481
|
+
root = ET.fromstring(xml_string)
|
|
2482
|
+
|
|
2483
|
+
texts = set()
|
|
2484
|
+
for elem in root.iter():
|
|
2485
|
+
text = elem.get('text', '').strip()
|
|
2486
|
+
desc = elem.get('content-desc', '').strip()
|
|
2487
|
+
# 只收集有意义的文本(长度2-30,非纯数字)
|
|
2488
|
+
for t in [text, desc]:
|
|
2489
|
+
if t and 2 <= len(t) <= 30 and not t.isdigit():
|
|
2490
|
+
texts.add(t)
|
|
2491
|
+
if len(texts) >= max_count * 2: # 收集足够后停止
|
|
2492
|
+
break
|
|
2493
|
+
|
|
2494
|
+
return list(texts)[:max_count]
|
|
2495
|
+
except Exception:
|
|
2496
|
+
return []
|
|
2497
|
+
|
|
2671
2498
|
def _has_business_id(self, resource_id: str) -> bool:
|
|
2672
2499
|
"""
|
|
2673
2500
|
判断resource_id是否是业务相关的ID
|
|
@@ -2699,6 +2526,68 @@ class BasicMobileToolsLite:
|
|
|
2699
2526
|
|
|
2700
2527
|
return True
|
|
2701
2528
|
|
|
2529
|
+
def _is_likely_clickable(self, class_name: str, resource_id: str, text: str,
|
|
2530
|
+
content_desc: str, clickable: bool, bounds: str) -> bool:
|
|
2531
|
+
"""
|
|
2532
|
+
启发式判断元素是否可能可点击
|
|
2533
|
+
|
|
2534
|
+
Android 的 clickable 属性经常不准确,因为:
|
|
2535
|
+
1. 点击事件可能设置在父容器上
|
|
2536
|
+
2. 使用 onTouchListener 而不是 onClick
|
|
2537
|
+
3. RecyclerView item 通过 ItemClickListener 处理
|
|
2538
|
+
|
|
2539
|
+
此方法通过多种规则推断元素的真实可点击性
|
|
2540
|
+
"""
|
|
2541
|
+
# 规则1:clickable=true 肯定可点击
|
|
2542
|
+
if clickable:
|
|
2543
|
+
return True
|
|
2544
|
+
|
|
2545
|
+
# 规则2:特定类型的控件通常可点击
|
|
2546
|
+
TYPICALLY_CLICKABLE = {
|
|
2547
|
+
'Button', 'ImageButton', 'CheckBox', 'RadioButton', 'Switch',
|
|
2548
|
+
'ToggleButton', 'FloatingActionButton', 'Chip', 'TabView',
|
|
2549
|
+
'EditText', 'TextInput', # 输入框可点击获取焦点
|
|
2550
|
+
}
|
|
2551
|
+
if class_name in TYPICALLY_CLICKABLE:
|
|
2552
|
+
return True
|
|
2553
|
+
|
|
2554
|
+
# 规则3:resource_id 包含可点击关键词
|
|
2555
|
+
if resource_id:
|
|
2556
|
+
id_lower = resource_id.lower()
|
|
2557
|
+
CLICK_KEYWORDS = [
|
|
2558
|
+
'btn', 'button', 'click', 'tap', 'submit', 'confirm',
|
|
2559
|
+
'cancel', 'close', 'back', 'next', 'prev', 'more',
|
|
2560
|
+
'action', 'link', 'menu', 'tab', 'item', 'cell',
|
|
2561
|
+
'card', 'avatar', 'icon', 'entry', 'option', 'arrow'
|
|
2562
|
+
]
|
|
2563
|
+
for kw in CLICK_KEYWORDS:
|
|
2564
|
+
if kw in id_lower:
|
|
2565
|
+
return True
|
|
2566
|
+
|
|
2567
|
+
# 规则4:content_desc 包含可点击暗示
|
|
2568
|
+
if content_desc:
|
|
2569
|
+
desc_lower = content_desc.lower()
|
|
2570
|
+
CLICK_HINTS = ['点击', '按钮', '关闭', '返回', '更多', 'click', 'tap', 'button', 'close']
|
|
2571
|
+
for hint in CLICK_HINTS:
|
|
2572
|
+
if hint in desc_lower:
|
|
2573
|
+
return True
|
|
2574
|
+
|
|
2575
|
+
# 规则5:有 resource_id 或 content_desc 的小图标可能可点击
|
|
2576
|
+
# (纯 ImageView 不加判断,误判率太高)
|
|
2577
|
+
if class_name in ('ImageView', 'Image') and (resource_id or content_desc) and bounds:
|
|
2578
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
|
|
2579
|
+
if match:
|
|
2580
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2581
|
+
w, h = x2 - x1, y2 - y1
|
|
2582
|
+
# 小图标(20-100px)更可能是按钮
|
|
2583
|
+
if 20 <= w <= 100 and 20 <= h <= 100:
|
|
2584
|
+
return True
|
|
2585
|
+
|
|
2586
|
+
# 规则6:移除(TextView 误判率太高,只依赖上面的规则)
|
|
2587
|
+
# 如果有 clickable=true 或 ID/desc 中有关键词,前面的规则已经覆盖
|
|
2588
|
+
|
|
2589
|
+
return False
|
|
2590
|
+
|
|
2702
2591
|
def find_close_button(self) -> Dict:
|
|
2703
2592
|
"""智能查找关闭按钮(不点击,只返回位置)
|
|
2704
2593
|
|
|
@@ -2712,7 +2601,7 @@ class BasicMobileToolsLite:
|
|
|
2712
2601
|
import re
|
|
2713
2602
|
|
|
2714
2603
|
if self._is_ios():
|
|
2715
|
-
return {"success": False, "
|
|
2604
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2716
2605
|
|
|
2717
2606
|
# 获取屏幕尺寸
|
|
2718
2607
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
@@ -2723,6 +2612,14 @@ class BasicMobileToolsLite:
|
|
|
2723
2612
|
import xml.etree.ElementTree as ET
|
|
2724
2613
|
root = ET.fromstring(xml_string)
|
|
2725
2614
|
|
|
2615
|
+
# 🔴 先检测是否有弹窗,避免误识别普通页面的按钮
|
|
2616
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
2617
|
+
root, screen_width, screen_height
|
|
2618
|
+
)
|
|
2619
|
+
|
|
2620
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
2621
|
+
return {"success": True, "popup": False}
|
|
2622
|
+
|
|
2726
2623
|
# 关闭按钮特征
|
|
2727
2624
|
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
|
|
2728
2625
|
candidates = []
|
|
@@ -2824,27 +2721,16 @@ class BasicMobileToolsLite:
|
|
|
2824
2721
|
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2825
2722
|
best = candidates[0]
|
|
2826
2723
|
|
|
2724
|
+
# Token 优化:只返回最必要的信息
|
|
2827
2725
|
return {
|
|
2828
2726
|
"success": True,
|
|
2829
|
-
"
|
|
2830
|
-
"
|
|
2831
|
-
|
|
2832
|
-
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
2833
|
-
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2834
|
-
"bounds": best['bounds'],
|
|
2835
|
-
"size": best['size'],
|
|
2836
|
-
"score": best['score']
|
|
2837
|
-
},
|
|
2838
|
-
"click_command": f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})",
|
|
2839
|
-
"other_candidates": [
|
|
2840
|
-
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
2841
|
-
for c in candidates[1:4]
|
|
2842
|
-
] if len(candidates) > 1 else [],
|
|
2843
|
-
"screen_size": {"width": screen_width, "height": screen_height}
|
|
2727
|
+
"popup": True,
|
|
2728
|
+
"close": {"x": best['x_percent'], "y": best['y_percent']},
|
|
2729
|
+
"cmd": f"click_by_percent({best['x_percent']},{best['y_percent']})"
|
|
2844
2730
|
}
|
|
2845
2731
|
|
|
2846
2732
|
except Exception as e:
|
|
2847
|
-
return {"success": False, "
|
|
2733
|
+
return {"success": False, "msg": str(e)}
|
|
2848
2734
|
|
|
2849
2735
|
def close_popup(self) -> Dict:
|
|
2850
2736
|
"""智能关闭弹窗(改进版)
|
|
@@ -2867,7 +2753,7 @@ class BasicMobileToolsLite:
|
|
|
2867
2753
|
|
|
2868
2754
|
# 获取屏幕尺寸
|
|
2869
2755
|
if self._is_ios():
|
|
2870
|
-
return {"success": False, "
|
|
2756
|
+
return {"success": False, "msg": "iOS暂不支持"}
|
|
2871
2757
|
|
|
2872
2758
|
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2873
2759
|
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
@@ -2895,8 +2781,10 @@ class BasicMobileToolsLite:
|
|
|
2895
2781
|
# 如果置信度不够高,记录但继续尝试查找关闭按钮
|
|
2896
2782
|
popup_detected = popup_bounds is not None and popup_confidence >= 0.6
|
|
2897
2783
|
|
|
2898
|
-
#
|
|
2899
|
-
#
|
|
2784
|
+
# 🔴 关键检查:如果没有检测到弹窗区域,直接返回"无弹窗"
|
|
2785
|
+
# 避免误点击普通页面上的"关闭"、"取消"等按钮
|
|
2786
|
+
if not popup_detected:
|
|
2787
|
+
return {"success": True, "popup": False}
|
|
2900
2788
|
|
|
2901
2789
|
# ===== 第二步:在弹窗范围内查找关闭按钮 =====
|
|
2902
2790
|
for idx, elem in enumerate(all_elements):
|
|
@@ -2905,7 +2793,6 @@ class BasicMobileToolsLite:
|
|
|
2905
2793
|
bounds_str = elem.attrib.get('bounds', '')
|
|
2906
2794
|
class_name = elem.attrib.get('class', '')
|
|
2907
2795
|
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2908
|
-
resource_id = elem.attrib.get('resource-id', '')
|
|
2909
2796
|
|
|
2910
2797
|
if not bounds_str:
|
|
2911
2798
|
continue
|
|
@@ -2922,7 +2809,7 @@ class BasicMobileToolsLite:
|
|
|
2922
2809
|
center_y = (y1 + y2) // 2
|
|
2923
2810
|
|
|
2924
2811
|
# 如果检测到弹窗区域,检查元素是否在弹窗范围内或附近
|
|
2925
|
-
in_popup =
|
|
2812
|
+
in_popup = True
|
|
2926
2813
|
popup_edge_bonus = 0
|
|
2927
2814
|
is_floating_close = False # 是否是浮动关闭按钮(在弹窗外部上方)
|
|
2928
2815
|
if popup_bounds:
|
|
@@ -2963,20 +2850,6 @@ class BasicMobileToolsLite:
|
|
|
2963
2850
|
# 浮动关闭按钮(在弹窗上方外侧)给予高额加分
|
|
2964
2851
|
if is_floating_close:
|
|
2965
2852
|
popup_edge_bonus += 5.0 # 大幅加分
|
|
2966
|
-
elif not popup_detected:
|
|
2967
|
-
# 没有检测到弹窗时,只处理有明确关闭特征的元素
|
|
2968
|
-
# 检查是否有明确的关闭特征(文本、resource-id、content-desc)
|
|
2969
|
-
has_explicit_close_feature = (
|
|
2970
|
-
text in close_texts or
|
|
2971
|
-
any(kw in content_desc.lower() for kw in close_desc_keywords) or
|
|
2972
|
-
'close' in resource_id.lower() or
|
|
2973
|
-
'dismiss' in resource_id.lower() or
|
|
2974
|
-
'cancel' in resource_id.lower()
|
|
2975
|
-
)
|
|
2976
|
-
if not has_explicit_close_feature:
|
|
2977
|
-
continue # 没有明确关闭特征,跳过
|
|
2978
|
-
# 有明确关闭特征时,允许处理
|
|
2979
|
-
in_popup = True
|
|
2980
2853
|
|
|
2981
2854
|
if not in_popup:
|
|
2982
2855
|
continue
|
|
@@ -3050,86 +2923,9 @@ class BasicMobileToolsLite:
|
|
|
3050
2923
|
pass
|
|
3051
2924
|
|
|
3052
2925
|
if not close_candidates:
|
|
3053
|
-
# 如果检测到高置信度的弹窗区域,先尝试点击常见的关闭按钮位置
|
|
3054
2926
|
if popup_detected and popup_bounds:
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
popup_height = py2 - py1
|
|
3058
|
-
|
|
3059
|
-
# 【优化】X按钮有三种常见位置:
|
|
3060
|
-
# 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
|
|
3061
|
-
# 2. 弹窗边界上方(浮动X按钮)
|
|
3062
|
-
# 3. 弹窗正下方(底部关闭按钮)
|
|
3063
|
-
offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
|
|
3064
|
-
offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
|
|
3065
|
-
offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
|
|
3066
|
-
|
|
3067
|
-
try_positions = [
|
|
3068
|
-
# 【最高优先级】弹窗内紧贴顶部边界
|
|
3069
|
-
(px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
|
|
3070
|
-
# 弹窗边界上方(浮动X按钮)
|
|
3071
|
-
(px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
|
|
3072
|
-
# 弹窗正下方中间(底部关闭按钮)
|
|
3073
|
-
((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
|
|
3074
|
-
# 弹窗正上方中间
|
|
3075
|
-
((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
|
|
3076
|
-
]
|
|
3077
|
-
|
|
3078
|
-
for try_x, try_y, position_name in try_positions:
|
|
3079
|
-
if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
|
|
3080
|
-
self.client.u2.click(try_x, try_y)
|
|
3081
|
-
time.sleep(0.3)
|
|
3082
|
-
|
|
3083
|
-
# 🎯 关键步骤:检查应用是否跳转,如果跳转说明弹窗去除失败,需要返回目标应用
|
|
3084
|
-
app_check = self._check_app_switched()
|
|
3085
|
-
return_result = None
|
|
3086
|
-
|
|
3087
|
-
if app_check['switched']:
|
|
3088
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
3089
|
-
return_result = self._return_to_target_app()
|
|
3090
|
-
|
|
3091
|
-
# 尝试后截图,让 AI 判断是否成功
|
|
3092
|
-
screenshot_result = self.take_screenshot("尝试关闭后")
|
|
3093
|
-
|
|
3094
|
-
msg = f"✅ 已尝试点击常见关闭按钮位置"
|
|
3095
|
-
if app_check['switched']:
|
|
3096
|
-
msg += f"\n⚠️ 应用已跳转,说明弹窗去除失败"
|
|
3097
|
-
if return_result:
|
|
3098
|
-
if return_result['success']:
|
|
3099
|
-
msg += f"\n{return_result['message']}"
|
|
3100
|
-
else:
|
|
3101
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
3102
|
-
|
|
3103
|
-
return {
|
|
3104
|
-
"success": True,
|
|
3105
|
-
"message": msg,
|
|
3106
|
-
"tried_positions": [p[2] for p in try_positions],
|
|
3107
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3108
|
-
"app_check": app_check,
|
|
3109
|
-
"return_to_app": return_result,
|
|
3110
|
-
"tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
|
|
3111
|
-
}
|
|
3112
|
-
|
|
3113
|
-
# 没有检测到弹窗区域,截图让 AI 分析
|
|
3114
|
-
screenshot_result = self.take_screenshot(description="页面截图", compress=True)
|
|
3115
|
-
|
|
3116
|
-
return {
|
|
3117
|
-
"success": False,
|
|
3118
|
-
"message": "❌ 未检测到弹窗区域,已截图供 AI 分析",
|
|
3119
|
-
"action_required": "请查看截图找到关闭按钮,调用 mobile_click_at_coords 点击",
|
|
3120
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3121
|
-
"screen_size": {"width": screen_width, "height": screen_height},
|
|
3122
|
-
"image_size": {
|
|
3123
|
-
"width": screenshot_result.get("image_width", screen_width),
|
|
3124
|
-
"height": screenshot_result.get("image_height", screen_height)
|
|
3125
|
-
},
|
|
3126
|
-
"original_size": {
|
|
3127
|
-
"width": screenshot_result.get("original_img_width", screen_width),
|
|
3128
|
-
"height": screenshot_result.get("original_img_height", screen_height)
|
|
3129
|
-
},
|
|
3130
|
-
"search_areas": ["弹窗右上角", "弹窗正上方", "弹窗下方中间", "屏幕右上角"],
|
|
3131
|
-
"time_warning": "⚠️ 截图分析期间弹窗可能自动消失。如果是定时弹窗,建议等待其自动消失。"
|
|
3132
|
-
}
|
|
2927
|
+
return {"success": False, "fallback": "vision", "popup": True}
|
|
2928
|
+
return {"success": True, "popup": False}
|
|
3133
2929
|
|
|
3134
2930
|
# 按得分排序,取最可能的
|
|
3135
2931
|
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
@@ -3147,62 +2943,22 @@ class BasicMobileToolsLite:
|
|
|
3147
2943
|
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
3148
2944
|
return_result = self._return_to_target_app()
|
|
3149
2945
|
|
|
3150
|
-
#
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
self._record_operation(
|
|
3155
|
-
'click',
|
|
3156
|
-
x=best['center_x'],
|
|
3157
|
-
y=best['center_y'],
|
|
3158
|
-
x_percent=best['x_percent'],
|
|
3159
|
-
y_percent=best['y_percent'],
|
|
3160
|
-
screen_width=screen_width,
|
|
3161
|
-
screen_height=screen_height,
|
|
3162
|
-
ref=f"close_popup_{best['position']}"
|
|
3163
|
-
)
|
|
2946
|
+
# 记录操作
|
|
2947
|
+
self._record_click('percent', f"{best['x_percent']}%,{best['y_percent']}%",
|
|
2948
|
+
best['x_percent'], best['y_percent'],
|
|
2949
|
+
element_desc=f"关闭按钮({best['position']})")
|
|
3164
2950
|
|
|
3165
|
-
#
|
|
3166
|
-
|
|
2951
|
+
# Token 优化:精简返回值
|
|
2952
|
+
result = {"success": True, "clicked": True}
|
|
3167
2953
|
if app_check['switched']:
|
|
3168
|
-
|
|
2954
|
+
result["switched"] = True
|
|
3169
2955
|
if return_result:
|
|
3170
|
-
|
|
3171
|
-
msg += f"\n{return_result['message']}"
|
|
3172
|
-
else:
|
|
3173
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
2956
|
+
result["returned"] = return_result['success']
|
|
3174
2957
|
|
|
3175
|
-
|
|
3176
|
-
# 如果弹窗还在,AI 可以选择点击其他候选按钮
|
|
3177
|
-
return {
|
|
3178
|
-
"success": True,
|
|
3179
|
-
"message": msg,
|
|
3180
|
-
"clicked": {
|
|
3181
|
-
"position": best['position'],
|
|
3182
|
-
"match_type": best['match_type'],
|
|
3183
|
-
"coords": (best['center_x'], best['center_y']),
|
|
3184
|
-
"percent": (best['x_percent'], best['y_percent'])
|
|
3185
|
-
},
|
|
3186
|
-
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
3187
|
-
"popup_detected": popup_detected,
|
|
3188
|
-
"popup_confidence": popup_confidence if popup_bounds else 0,
|
|
3189
|
-
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_detected else None,
|
|
3190
|
-
"app_check": app_check,
|
|
3191
|
-
"return_to_app": return_result,
|
|
3192
|
-
"other_candidates": [
|
|
3193
|
-
{
|
|
3194
|
-
"position": c['position'],
|
|
3195
|
-
"type": c['match_type'],
|
|
3196
|
-
"coords": (c['center_x'], c['center_y']),
|
|
3197
|
-
"percent": (c['x_percent'], c['y_percent'])
|
|
3198
|
-
}
|
|
3199
|
-
for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
|
|
3200
|
-
],
|
|
3201
|
-
"tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置"
|
|
3202
|
-
}
|
|
2958
|
+
return result
|
|
3203
2959
|
|
|
3204
2960
|
except Exception as e:
|
|
3205
|
-
return {"success": False, "
|
|
2961
|
+
return {"success": False, "msg": str(e)}
|
|
3206
2962
|
|
|
3207
2963
|
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
3208
2964
|
"""根据相对坐标获取位置名称"""
|
|
@@ -3284,15 +3040,6 @@ class BasicMobileToolsLite:
|
|
|
3284
3040
|
resource_id = elem.attrib.get('resource-id', '')
|
|
3285
3041
|
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
3286
3042
|
|
|
3287
|
-
# 检查是否是关闭按钮
|
|
3288
|
-
is_close_button = (
|
|
3289
|
-
'close' in resource_id.lower() or
|
|
3290
|
-
'dismiss' in resource_id.lower() or
|
|
3291
|
-
'cancel' in resource_id.lower() or
|
|
3292
|
-
'×' in elem.attrib.get('text', '') or
|
|
3293
|
-
'X' in elem.attrib.get('text', '')
|
|
3294
|
-
)
|
|
3295
|
-
|
|
3296
3043
|
all_elements.append({
|
|
3297
3044
|
'idx': idx,
|
|
3298
3045
|
'bounds': (x1, y1, x2, y2),
|
|
@@ -3305,7 +3052,6 @@ class BasicMobileToolsLite:
|
|
|
3305
3052
|
'clickable': clickable,
|
|
3306
3053
|
'center_x': (x1 + x2) // 2,
|
|
3307
3054
|
'center_y': (y1 + y2) // 2,
|
|
3308
|
-
'is_close_button': is_close_button,
|
|
3309
3055
|
})
|
|
3310
3056
|
|
|
3311
3057
|
if not all_elements:
|
|
@@ -3314,8 +3060,6 @@ class BasicMobileToolsLite:
|
|
|
3314
3060
|
# 弹窗检测关键词
|
|
3315
3061
|
dialog_class_keywords = ['Dialog', 'Popup', 'Alert', 'Modal', 'BottomSheet', 'PopupWindow']
|
|
3316
3062
|
dialog_id_keywords = ['dialog', 'popup', 'alert', 'modal', 'bottom_sheet', 'overlay', 'mask']
|
|
3317
|
-
# 广告弹窗关键词(全屏广告、激励视频等)
|
|
3318
|
-
ad_popup_keywords = ['ad_close', 'ad_button', 'full_screen', 'interstitial', 'reward', 'close_icon', 'close_btn']
|
|
3319
3063
|
|
|
3320
3064
|
popup_candidates = []
|
|
3321
3065
|
has_mask_layer = False
|
|
@@ -3346,59 +3090,6 @@ class BasicMobileToolsLite:
|
|
|
3346
3090
|
if y1 < 50:
|
|
3347
3091
|
continue
|
|
3348
3092
|
|
|
3349
|
-
# 【非弹窗特征】如果元素包含底部导航栏(底部tab),则不是弹窗
|
|
3350
|
-
# 底部导航栏通常在屏幕底部,高度约100-200像素
|
|
3351
|
-
if y2 > screen_height * 0.85:
|
|
3352
|
-
# 检查是否包含tab相关的resource-id或class
|
|
3353
|
-
if 'tab' in resource_id.lower() or 'Tab' in class_name or 'navigation' in resource_id.lower():
|
|
3354
|
-
continue # 跳过底部导航栏
|
|
3355
|
-
|
|
3356
|
-
# 【非弹窗特征】如果元素包含顶部搜索栏,则不是弹窗
|
|
3357
|
-
if y1 < screen_height * 0.15:
|
|
3358
|
-
if 'search' in resource_id.lower() or 'Search' in class_name:
|
|
3359
|
-
continue # 跳过顶部搜索栏
|
|
3360
|
-
|
|
3361
|
-
# 先检查是否有强弹窗特征(用于后续判断)
|
|
3362
|
-
has_strong_popup_feature = (
|
|
3363
|
-
any(kw in class_name for kw in dialog_class_keywords) or
|
|
3364
|
-
any(kw in resource_id.lower() for kw in dialog_id_keywords) or
|
|
3365
|
-
any(kw in resource_id.lower() for kw in ad_popup_keywords) # 广告弹窗关键词
|
|
3366
|
-
)
|
|
3367
|
-
|
|
3368
|
-
# 检查是否有子元素是关闭按钮(作为弹窗特征)
|
|
3369
|
-
has_close_button_child = False
|
|
3370
|
-
elem_bounds = elem['bounds']
|
|
3371
|
-
for other_elem in all_elements:
|
|
3372
|
-
if other_elem['idx'] == elem['idx']:
|
|
3373
|
-
continue
|
|
3374
|
-
if other_elem['is_close_button']:
|
|
3375
|
-
# 检查关闭按钮是否在这个元素范围内
|
|
3376
|
-
ox1, oy1, ox2, oy2 = other_elem['bounds']
|
|
3377
|
-
ex1, ey1, ex2, ey2 = elem_bounds
|
|
3378
|
-
if ex1 <= ox1 and ey1 <= oy1 and ex2 >= ox2 and ey2 >= oy2:
|
|
3379
|
-
has_close_button_child = True
|
|
3380
|
-
break
|
|
3381
|
-
|
|
3382
|
-
# 【非弹窗特征】如果元素包含明显的页面内容特征,则不是弹窗
|
|
3383
|
-
# 检查是否包含视频播放器、内容列表等页面元素
|
|
3384
|
-
page_content_keywords = ['video', 'player', 'recycler', 'list', 'scroll', 'viewpager', 'fragment']
|
|
3385
|
-
if any(kw in resource_id.lower() or kw in class_name.lower() for kw in page_content_keywords):
|
|
3386
|
-
# 如果面积很大且没有强弹窗特征,则不是弹窗
|
|
3387
|
-
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
3388
|
-
continue
|
|
3389
|
-
|
|
3390
|
-
# 【非弹窗特征】如果元素面积过大(接近全屏),即使居中也不应该是弹窗
|
|
3391
|
-
# 真正的弹窗通常不会超过屏幕的60%
|
|
3392
|
-
# 对于面积 > 0.6 的元素,如果没有强特征,直接跳过(避免误判首页内容区域)
|
|
3393
|
-
if area_ratio > 0.6 and not has_strong_popup_feature:
|
|
3394
|
-
continue # 跳过大面积非弹窗元素(接近全屏的内容区域,如首页视频播放区域)
|
|
3395
|
-
|
|
3396
|
-
# 对于面积 > 0.7 的元素,即使有强特征也要更严格
|
|
3397
|
-
if area_ratio > 0.7:
|
|
3398
|
-
# 需要非常强的特征才认为是弹窗
|
|
3399
|
-
if not has_strong_popup_feature:
|
|
3400
|
-
continue
|
|
3401
|
-
|
|
3402
3093
|
confidence = 0.0
|
|
3403
3094
|
|
|
3404
3095
|
# 【强特征】class 名称包含弹窗关键词 (+0.5)
|
|
@@ -3409,46 +3100,19 @@ class BasicMobileToolsLite:
|
|
|
3409
3100
|
if any(kw in resource_id.lower() for kw in dialog_id_keywords):
|
|
3410
3101
|
confidence += 0.4
|
|
3411
3102
|
|
|
3412
|
-
# 【强特征】resource-id 包含广告弹窗关键词 (+0.4)
|
|
3413
|
-
if any(kw in resource_id.lower() for kw in ad_popup_keywords):
|
|
3414
|
-
confidence += 0.4
|
|
3415
|
-
|
|
3416
|
-
# 【强特征】包含关闭按钮作为子元素 (+0.3)
|
|
3417
|
-
if has_close_button_child:
|
|
3418
|
-
confidence += 0.3
|
|
3419
|
-
|
|
3420
3103
|
# 【中等特征】居中显示 (+0.2)
|
|
3421
|
-
# 但如果没有强特征,降低权重
|
|
3422
3104
|
center_x = elem['center_x']
|
|
3423
3105
|
center_y = elem['center_y']
|
|
3424
3106
|
is_centered_x = abs(center_x - screen_width / 2) < screen_width * 0.15
|
|
3425
3107
|
is_centered_y = abs(center_y - screen_height / 2) < screen_height * 0.25
|
|
3426
|
-
|
|
3427
|
-
has_strong_feature = (
|
|
3428
|
-
any(kw in class_name for kw in dialog_class_keywords) or
|
|
3429
|
-
any(kw in resource_id.lower() for kw in dialog_id_keywords) or
|
|
3430
|
-
any(kw in resource_id.lower() for kw in ad_popup_keywords) or
|
|
3431
|
-
has_close_button_child
|
|
3432
|
-
)
|
|
3433
|
-
|
|
3434
3108
|
if is_centered_x and is_centered_y:
|
|
3435
|
-
|
|
3436
|
-
confidence += 0.2
|
|
3437
|
-
else:
|
|
3438
|
-
confidence += 0.1 # 没有强特征时降低权重
|
|
3109
|
+
confidence += 0.2
|
|
3439
3110
|
elif is_centered_x:
|
|
3440
|
-
|
|
3441
|
-
confidence += 0.1
|
|
3442
|
-
else:
|
|
3443
|
-
confidence += 0.05 # 没有强特征时降低权重
|
|
3111
|
+
confidence += 0.1
|
|
3444
3112
|
|
|
3445
3113
|
# 【中等特征】非全屏但有一定大小 (+0.15)
|
|
3446
|
-
# 但如果没有强特征,降低权重
|
|
3447
3114
|
if 0.15 < area_ratio < 0.75:
|
|
3448
|
-
|
|
3449
|
-
confidence += 0.15
|
|
3450
|
-
else:
|
|
3451
|
-
confidence += 0.08 # 没有强特征时降低权重
|
|
3115
|
+
confidence += 0.15
|
|
3452
3116
|
|
|
3453
3117
|
# 【弱特征】XML 顺序靠后(在视图层级上层)(+0.1)
|
|
3454
3118
|
if elem['idx'] > len(all_elements) * 0.5:
|
|
@@ -3475,22 +3139,8 @@ class BasicMobileToolsLite:
|
|
|
3475
3139
|
popup_candidates.sort(key=lambda x: (x['confidence'], x['idx']), reverse=True)
|
|
3476
3140
|
best = popup_candidates[0]
|
|
3477
3141
|
|
|
3478
|
-
#
|
|
3479
|
-
|
|
3480
|
-
has_strong_feature = (
|
|
3481
|
-
any(kw in best['class'] for kw in dialog_class_keywords) or
|
|
3482
|
-
any(kw in best['resource_id'].lower() for kw in dialog_id_keywords) or
|
|
3483
|
-
any(kw in best['resource_id'].lower() for kw in ad_popup_keywords)
|
|
3484
|
-
)
|
|
3485
|
-
|
|
3486
|
-
if has_strong_feature:
|
|
3487
|
-
# 有强特征时,阈值0.7
|
|
3488
|
-
threshold = 0.7
|
|
3489
|
-
else:
|
|
3490
|
-
# 没有强特征时,阈值0.85(更严格)
|
|
3491
|
-
threshold = 0.85
|
|
3492
|
-
|
|
3493
|
-
if best['confidence'] >= threshold:
|
|
3142
|
+
# 只有置信度 >= 0.6 才返回弹窗
|
|
3143
|
+
if best['confidence'] >= 0.6:
|
|
3494
3144
|
return best['bounds'], best['confidence']
|
|
3495
3145
|
|
|
3496
3146
|
return None, best['confidence']
|
|
@@ -4209,10 +3859,28 @@ class BasicMobileToolsLite:
|
|
|
4209
3859
|
try:
|
|
4210
3860
|
import xml.etree.ElementTree as ET
|
|
4211
3861
|
|
|
4212
|
-
# ========== 第
|
|
3862
|
+
# ========== 第0步:先检测是否有弹窗 ==========
|
|
4213
3863
|
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
4214
3864
|
root = ET.fromstring(xml_string)
|
|
4215
3865
|
|
|
3866
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3867
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
3868
|
+
|
|
3869
|
+
popup_bounds, popup_confidence = self._detect_popup_with_confidence(
|
|
3870
|
+
root, screen_width, screen_height
|
|
3871
|
+
)
|
|
3872
|
+
|
|
3873
|
+
# 如果没有检测到弹窗,直接返回"无弹窗"
|
|
3874
|
+
if popup_bounds is None or popup_confidence < 0.5:
|
|
3875
|
+
result["success"] = True
|
|
3876
|
+
result["method"] = None
|
|
3877
|
+
result["message"] = "ℹ️ 当前页面未检测到弹窗,无需关闭"
|
|
3878
|
+
result["popup_detected"] = False
|
|
3879
|
+
result["popup_confidence"] = popup_confidence
|
|
3880
|
+
return result
|
|
3881
|
+
|
|
3882
|
+
# ========== 第1步:控件树查找关闭按钮 ==========
|
|
3883
|
+
|
|
4216
3884
|
# 关闭按钮的常见特征
|
|
4217
3885
|
close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
|
|
4218
3886
|
close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
|
|
@@ -4291,12 +3959,6 @@ class BasicMobileToolsLite:
|
|
|
4291
3959
|
cx, cy = best['center']
|
|
4292
3960
|
bounds = best['bounds']
|
|
4293
3961
|
|
|
4294
|
-
# 点击前截图(用于自动学习)
|
|
4295
|
-
pre_screenshot = None
|
|
4296
|
-
if auto_learn:
|
|
4297
|
-
pre_result = self.take_screenshot(description="关闭前", compress=False)
|
|
4298
|
-
pre_screenshot = pre_result.get("screenshot_path")
|
|
4299
|
-
|
|
4300
3962
|
# 点击(click_at_coords 内部已包含应用状态检查和自动返回)
|
|
4301
3963
|
click_result = self.click_at_coords(cx, cy)
|
|
4302
3964
|
time.sleep(0.5)
|
|
@@ -4326,17 +3988,11 @@ class BasicMobileToolsLite:
|
|
|
4326
3988
|
result["message"] = msg
|
|
4327
3989
|
result["app_check"] = app_check
|
|
4328
3990
|
result["return_to_app"] = return_result
|
|
4329
|
-
|
|
4330
|
-
# 自动学习:检查这个 X 是否已在模板库,不在就添加
|
|
4331
|
-
if auto_learn and pre_screenshot:
|
|
4332
|
-
learn_result = self._auto_learn_template(pre_screenshot, bounds)
|
|
4333
|
-
if learn_result:
|
|
4334
|
-
result["learned_template"] = learn_result
|
|
4335
|
-
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
3991
|
+
result["tip"] = "💡 建议调用 mobile_screenshot_with_som 确认弹窗是否已关闭"
|
|
4336
3992
|
|
|
4337
3993
|
return result
|
|
4338
3994
|
|
|
4339
|
-
# ========== 第2
|
|
3995
|
+
# ========== 第2步:模板匹配(自动执行,不需要 AI 介入)==========
|
|
4340
3996
|
screenshot_path = None
|
|
4341
3997
|
try:
|
|
4342
3998
|
from .template_matcher import TemplateMatcher
|
|
@@ -4355,16 +4011,14 @@ class BasicMobileToolsLite:
|
|
|
4355
4011
|
x_pct = best["percent"]["x"]
|
|
4356
4012
|
y_pct = best["percent"]["y"]
|
|
4357
4013
|
|
|
4358
|
-
#
|
|
4014
|
+
# 点击
|
|
4359
4015
|
click_result = self.click_by_percent(x_pct, y_pct)
|
|
4360
4016
|
time.sleep(0.5)
|
|
4361
4017
|
|
|
4362
|
-
# 🎯 再次检查应用状态(确保弹窗去除没有导致应用跳转)
|
|
4363
4018
|
app_check = self._check_app_switched()
|
|
4364
4019
|
return_result = None
|
|
4365
4020
|
|
|
4366
4021
|
if app_check['switched']:
|
|
4367
|
-
# 应用已跳转,说明弹窗去除失败,尝试返回目标应用
|
|
4368
4022
|
return_result = self._return_to_target_app()
|
|
4369
4023
|
|
|
4370
4024
|
result["success"] = True
|
|
@@ -4375,12 +4029,9 @@ class BasicMobileToolsLite:
|
|
|
4375
4029
|
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
4376
4030
|
|
|
4377
4031
|
if app_check['switched']:
|
|
4378
|
-
msg += f"\n⚠️
|
|
4032
|
+
msg += f"\n⚠️ 应用已跳转"
|
|
4379
4033
|
if return_result:
|
|
4380
|
-
|
|
4381
|
-
msg += f"\n{return_result['message']}"
|
|
4382
|
-
else:
|
|
4383
|
-
msg += f"\n❌ 自动返回失败: {return_result['message']}"
|
|
4034
|
+
msg += f"\n{return_result['message']}"
|
|
4384
4035
|
|
|
4385
4036
|
result["message"] = msg
|
|
4386
4037
|
result["app_check"] = app_check
|
|
@@ -4392,17 +4043,12 @@ class BasicMobileToolsLite:
|
|
|
4392
4043
|
except Exception:
|
|
4393
4044
|
pass # 模板匹配失败,继续下一步
|
|
4394
4045
|
|
|
4395
|
-
# ========== 第3
|
|
4396
|
-
if not screenshot_path:
|
|
4397
|
-
screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
|
|
4398
|
-
|
|
4046
|
+
# ========== 第3步:控件树和模板匹配都失败,提示 AI 使用视觉识别 ==========
|
|
4399
4047
|
result["success"] = False
|
|
4048
|
+
result["fallback"] = "vision"
|
|
4400
4049
|
result["method"] = None
|
|
4401
|
-
result["
|
|
4402
|
-
|
|
4403
|
-
"💡 找到后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
4404
|
-
result["screenshot"] = screenshot_result if not screenshot_path else {"screenshot_path": screenshot_path}
|
|
4405
|
-
result["need_ai_analysis"] = True
|
|
4050
|
+
result["popup_detected"] = True
|
|
4051
|
+
result["message"] = "⚠️ 控件树和模板匹配都未找到关闭按钮,请调用 mobile_screenshot_with_som 截图后用 click_by_som 点击"
|
|
4406
4052
|
|
|
4407
4053
|
return result
|
|
4408
4054
|
|