mobile-mcp-ai 2.1.2__py3-none-any.whl → 2.5.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mobile_mcp/__init__.py +34 -0
- mobile_mcp/config.py +142 -0
- mobile_mcp/core/basic_tools_lite.py +3266 -0
- {core → mobile_mcp/core}/device_manager.py +2 -2
- mobile_mcp/core/dynamic_config.py +272 -0
- mobile_mcp/core/ios_client_wda.py +569 -0
- mobile_mcp/core/ios_device_manager_wda.py +306 -0
- {core → mobile_mcp/core}/mobile_client.py +279 -39
- mobile_mcp/core/template_matcher.py +429 -0
- mobile_mcp/core/templates/close_buttons/auto_x_0112_151217.png +0 -0
- mobile_mcp/core/templates/close_buttons/auto_x_0112_152037.png +0 -0
- mobile_mcp/core/templates/close_buttons/auto_x_0112_152840.png +0 -0
- mobile_mcp/core/templates/close_buttons/auto_x_0112_153256.png +0 -0
- mobile_mcp/core/templates/close_buttons/auto_x_0112_154847.png +0 -0
- mobile_mcp/core/templates/close_buttons/gray_x_stock_ad.png +0 -0
- {core → mobile_mcp/core}/utils/smart_wait.py +3 -3
- mobile_mcp/mcp_tools/__init__.py +10 -0
- mobile_mcp/mcp_tools/mcp_server.py +1071 -0
- mobile_mcp_ai-2.5.8.dist-info/METADATA +469 -0
- mobile_mcp_ai-2.5.8.dist-info/RECORD +32 -0
- mobile_mcp_ai-2.5.8.dist-info/entry_points.txt +2 -0
- mobile_mcp_ai-2.5.8.dist-info/licenses/LICENSE +201 -0
- mobile_mcp_ai-2.5.8.dist-info/top_level.txt +1 -0
- core/ai/__init__.py +0 -11
- core/ai/ai_analyzer.py +0 -197
- core/ai/ai_config.py +0 -116
- core/ai/ai_platform_adapter.py +0 -399
- core/ai/smart_test_executor.py +0 -520
- core/ai/test_generator.py +0 -365
- core/ai/test_generator_from_history.py +0 -391
- core/ai/test_generator_standalone.py +0 -293
- core/assertion/__init__.py +0 -9
- core/assertion/smart_assertion.py +0 -341
- core/basic_tools.py +0 -377
- core/h5/__init__.py +0 -10
- core/h5/h5_handler.py +0 -548
- core/ios_client.py +0 -219
- core/ios_device_manager.py +0 -252
- core/locator/__init__.py +0 -10
- core/locator/cursor_ai_auto_analyzer.py +0 -119
- core/locator/cursor_vision_helper.py +0 -414
- core/locator/mobile_smart_locator.py +0 -1640
- core/locator/position_analyzer.py +0 -813
- core/locator/script_updater.py +0 -157
- core/nl_test_runner.py +0 -585
- core/smart_app_launcher.py +0 -334
- core/smart_tools.py +0 -311
- mcp/__init__.py +0 -8
- mcp/mcp_server.py +0 -1919
- mcp/mcp_server_simple.py +0 -476
- mobile_mcp_ai-2.1.2.dist-info/METADATA +0 -567
- mobile_mcp_ai-2.1.2.dist-info/RECORD +0 -45
- mobile_mcp_ai-2.1.2.dist-info/entry_points.txt +0 -2
- mobile_mcp_ai-2.1.2.dist-info/top_level.txt +0 -4
- vision/__init__.py +0 -10
- vision/vision_locator.py +0 -404
- {core → mobile_mcp/core}/__init__.py +0 -0
- {core → mobile_mcp/core}/utils/__init__.py +0 -0
- {core → mobile_mcp/core}/utils/logger.py +0 -0
- {core → mobile_mcp/core}/utils/operation_history_manager.py +0 -0
- {utils → mobile_mcp/utils}/__init__.py +0 -0
- {utils → mobile_mcp/utils}/logger.py +0 -0
- {utils → mobile_mcp/utils}/xml_formatter.py +0 -0
- {utils → mobile_mcp/utils}/xml_parser.py +0 -0
- {mobile_mcp_ai-2.1.2.dist-info → mobile_mcp_ai-2.5.8.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,3266 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
精简版基础工具 - 纯 MCP,依赖 Cursor 视觉能力
|
|
5
|
+
|
|
6
|
+
特点:
|
|
7
|
+
- 不需要 AI 密钥
|
|
8
|
+
- 核心功能精简
|
|
9
|
+
- 保留 pytest 脚本生成
|
|
10
|
+
- 支持操作历史记录
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import time
|
|
15
|
+
import re
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Dict, List, Optional
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BasicMobileToolsLite:
|
|
22
|
+
"""精简版移动端工具"""
|
|
23
|
+
|
|
24
|
+
def __init__(self, mobile_client):
|
|
25
|
+
self.client = mobile_client
|
|
26
|
+
|
|
27
|
+
# 截图目录
|
|
28
|
+
project_root = Path(__file__).parent.parent
|
|
29
|
+
self.screenshot_dir = project_root / "screenshots"
|
|
30
|
+
self.screenshot_dir.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
|
|
32
|
+
# 操作历史(用于生成 pytest 脚本)
|
|
33
|
+
self.operation_history: List[Dict] = []
|
|
34
|
+
|
|
35
|
+
def _is_ios(self) -> bool:
|
|
36
|
+
"""判断当前是否为 iOS 平台"""
|
|
37
|
+
return getattr(self.client, 'platform', 'android') == 'ios'
|
|
38
|
+
|
|
39
|
+
def _get_ios_client(self):
|
|
40
|
+
"""获取 iOS 客户端"""
|
|
41
|
+
if hasattr(self.client, '_ios_client') and self.client._ios_client:
|
|
42
|
+
return self.client._ios_client
|
|
43
|
+
if hasattr(self.client, 'wda') and self.client.wda:
|
|
44
|
+
return self.client.wda
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
def _record_operation(self, action: str, **kwargs):
|
|
48
|
+
"""记录操作到历史"""
|
|
49
|
+
record = {
|
|
50
|
+
'action': action,
|
|
51
|
+
'timestamp': datetime.now().isoformat(),
|
|
52
|
+
**kwargs
|
|
53
|
+
}
|
|
54
|
+
self.operation_history.append(record)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ==================== 截图 ====================
|
|
59
|
+
|
|
60
|
+
def take_screenshot(self, description: str = "", compress: bool = True,
|
|
61
|
+
max_width: int = 720, quality: int = 75,
|
|
62
|
+
crop_x: int = 0, crop_y: int = 0, crop_size: int = 0) -> Dict:
|
|
63
|
+
"""截图(支持压缩和局部裁剪)
|
|
64
|
+
|
|
65
|
+
压缩原理:
|
|
66
|
+
1. 先截取原始 PNG 图片
|
|
67
|
+
2. 缩小尺寸(如 1080p → 720p)
|
|
68
|
+
3. 转换为 JPEG 格式 + 降低质量(如 100% → 75%)
|
|
69
|
+
4. 最终文件从 2MB 压缩到约 80KB(节省 96%)
|
|
70
|
+
|
|
71
|
+
局部裁剪(用于精确识别小元素):
|
|
72
|
+
- 第一次全屏截图,AI 返回大概坐标
|
|
73
|
+
- 第二次传入 crop_x, crop_y, crop_size 截取局部区域
|
|
74
|
+
- 局部区域不压缩,保持清晰度,AI 可精确识别
|
|
75
|
+
- 返回 crop_offset_x/y 用于坐标换算
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
description: 截图描述(可选)
|
|
79
|
+
compress: 是否压缩(默认 True,推荐开启省 token)
|
|
80
|
+
max_width: 压缩后最大宽度(默认 720,对 AI 识别足够)
|
|
81
|
+
quality: JPEG 质量 1-100(默认 75,肉眼几乎看不出区别)
|
|
82
|
+
crop_x: 裁剪中心点 X 坐标(屏幕坐标,0 表示不裁剪)
|
|
83
|
+
crop_y: 裁剪中心点 Y 坐标(屏幕坐标,0 表示不裁剪)
|
|
84
|
+
crop_size: 裁剪区域大小(默认 0 不裁剪,推荐 200-400)
|
|
85
|
+
|
|
86
|
+
压缩效果示例:
|
|
87
|
+
原图 PNG: 2048KB
|
|
88
|
+
压缩后 JPEG (720p, 75%): ~80KB
|
|
89
|
+
节省: 96%
|
|
90
|
+
"""
|
|
91
|
+
try:
|
|
92
|
+
from PIL import Image
|
|
93
|
+
|
|
94
|
+
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
95
|
+
platform = "ios" if self._is_ios() else "android"
|
|
96
|
+
|
|
97
|
+
# 第1步:截图保存为临时 PNG
|
|
98
|
+
temp_filename = f"temp_{timestamp}.png"
|
|
99
|
+
temp_path = self.screenshot_dir / temp_filename
|
|
100
|
+
|
|
101
|
+
# 获取屏幕尺寸并截图
|
|
102
|
+
screen_width, screen_height = 0, 0
|
|
103
|
+
if self._is_ios():
|
|
104
|
+
ios_client = self._get_ios_client()
|
|
105
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
106
|
+
ios_client.wda.screenshot(str(temp_path))
|
|
107
|
+
size = ios_client.wda.window_size()
|
|
108
|
+
screen_width, screen_height = size[0], size[1]
|
|
109
|
+
else:
|
|
110
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
111
|
+
else:
|
|
112
|
+
self.client.u2.screenshot(str(temp_path))
|
|
113
|
+
info = self.client.u2.info
|
|
114
|
+
screen_width = info.get('displayWidth', 0)
|
|
115
|
+
screen_height = info.get('displayHeight', 0)
|
|
116
|
+
|
|
117
|
+
original_size = temp_path.stat().st_size
|
|
118
|
+
|
|
119
|
+
# 第2步:打开图片
|
|
120
|
+
img = Image.open(temp_path)
|
|
121
|
+
|
|
122
|
+
# 第2.5步:局部裁剪(如果指定了裁剪参数)
|
|
123
|
+
crop_offset_x, crop_offset_y = 0, 0
|
|
124
|
+
is_cropped = False
|
|
125
|
+
|
|
126
|
+
if crop_x > 0 and crop_y > 0 and crop_size > 0:
|
|
127
|
+
# 计算裁剪区域(以 crop_x, crop_y 为中心)
|
|
128
|
+
half_size = crop_size // 2
|
|
129
|
+
left = max(0, crop_x - half_size)
|
|
130
|
+
top = max(0, crop_y - half_size)
|
|
131
|
+
right = min(img.width, crop_x + half_size)
|
|
132
|
+
bottom = min(img.height, crop_y + half_size)
|
|
133
|
+
|
|
134
|
+
# 记录偏移量(用于坐标换算)
|
|
135
|
+
crop_offset_x = left
|
|
136
|
+
crop_offset_y = top
|
|
137
|
+
|
|
138
|
+
# 裁剪
|
|
139
|
+
img = img.crop((left, top, right, bottom))
|
|
140
|
+
is_cropped = True
|
|
141
|
+
|
|
142
|
+
# ========== 情况1:局部裁剪截图(不压缩,保持清晰度)==========
|
|
143
|
+
if is_cropped:
|
|
144
|
+
# 生成文件名
|
|
145
|
+
if description:
|
|
146
|
+
safe_desc = re.sub(r'[^\w\s-]', '', description).strip().replace(' ', '_')
|
|
147
|
+
filename = f"screenshot_{platform}_crop_{safe_desc}_{timestamp}.png"
|
|
148
|
+
else:
|
|
149
|
+
filename = f"screenshot_{platform}_crop_{timestamp}.png"
|
|
150
|
+
|
|
151
|
+
final_path = self.screenshot_dir / filename
|
|
152
|
+
|
|
153
|
+
# 保存为 PNG(保持清晰度)
|
|
154
|
+
img.save(str(final_path), "PNG")
|
|
155
|
+
|
|
156
|
+
# 删除临时文件
|
|
157
|
+
temp_path.unlink()
|
|
158
|
+
|
|
159
|
+
cropped_size = final_path.stat().st_size
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
"success": True,
|
|
163
|
+
"screenshot_path": str(final_path),
|
|
164
|
+
"screen_width": screen_width,
|
|
165
|
+
"screen_height": screen_height,
|
|
166
|
+
"image_width": img.width,
|
|
167
|
+
"image_height": img.height,
|
|
168
|
+
"crop_offset_x": crop_offset_x,
|
|
169
|
+
"crop_offset_y": crop_offset_y,
|
|
170
|
+
"file_size": f"{cropped_size/1024:.1f}KB",
|
|
171
|
+
"message": f"🔍 局部截图已保存: {final_path}\n"
|
|
172
|
+
f"📐 裁剪区域: ({crop_offset_x}, {crop_offset_y}) 起,{img.width}x{img.height} 像素\n"
|
|
173
|
+
f"📦 文件大小: {cropped_size/1024:.0f}KB\n"
|
|
174
|
+
f"🎯 【坐标换算】AI 返回坐标 (x, y) 后:\n"
|
|
175
|
+
f" 实际屏幕坐标 = ({crop_offset_x} + x, {crop_offset_y} + y)\n"
|
|
176
|
+
f" 或直接调用 mobile_click_at_coords(x, y, crop_offset_x={crop_offset_x}, crop_offset_y={crop_offset_y})"
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
# ========== 情况2:全屏压缩截图 ==========
|
|
180
|
+
elif compress:
|
|
181
|
+
# 🔴 关键:记录原始图片尺寸(用于坐标转换)
|
|
182
|
+
# 注意:截图尺寸可能和 u2.info 的 displayWidth 不一致!
|
|
183
|
+
original_img_width = img.width
|
|
184
|
+
original_img_height = img.height
|
|
185
|
+
|
|
186
|
+
# 第3步:缩小尺寸(保持宽高比)
|
|
187
|
+
image_width, image_height = img.width, img.height
|
|
188
|
+
|
|
189
|
+
if img.width > max_width:
|
|
190
|
+
ratio = max_width / img.width
|
|
191
|
+
new_w = max_width
|
|
192
|
+
new_h = int(img.height * ratio)
|
|
193
|
+
# 兼容不同版本的 Pillow
|
|
194
|
+
try:
|
|
195
|
+
resample = Image.Resampling.LANCZOS
|
|
196
|
+
except AttributeError:
|
|
197
|
+
try:
|
|
198
|
+
resample = Image.LANCZOS
|
|
199
|
+
except AttributeError:
|
|
200
|
+
resample = Image.ANTIALIAS
|
|
201
|
+
img = img.resize((new_w, new_h), resample)
|
|
202
|
+
image_width, image_height = new_w, new_h
|
|
203
|
+
|
|
204
|
+
# 生成文件名(JPEG 格式)
|
|
205
|
+
if description:
|
|
206
|
+
safe_desc = re.sub(r'[^\w\s-]', '', description).strip().replace(' ', '_')
|
|
207
|
+
filename = f"screenshot_{platform}_{safe_desc}_{timestamp}.jpg"
|
|
208
|
+
else:
|
|
209
|
+
filename = f"screenshot_{platform}_{timestamp}.jpg"
|
|
210
|
+
|
|
211
|
+
final_path = self.screenshot_dir / filename
|
|
212
|
+
|
|
213
|
+
# 保存为 JPEG(处理透明通道)
|
|
214
|
+
if img.mode in ('RGBA', 'LA', 'P'):
|
|
215
|
+
background = Image.new('RGB', img.size, (255, 255, 255))
|
|
216
|
+
if img.mode == 'P':
|
|
217
|
+
img = img.convert('RGBA')
|
|
218
|
+
background.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None)
|
|
219
|
+
img = background
|
|
220
|
+
elif img.mode != 'RGB':
|
|
221
|
+
img = img.convert("RGB")
|
|
222
|
+
|
|
223
|
+
img.save(str(final_path), "JPEG", quality=quality)
|
|
224
|
+
temp_path.unlink()
|
|
225
|
+
|
|
226
|
+
compressed_size = final_path.stat().st_size
|
|
227
|
+
saved_percent = (1 - compressed_size / original_size) * 100
|
|
228
|
+
|
|
229
|
+
return {
|
|
230
|
+
"success": True,
|
|
231
|
+
"screenshot_path": str(final_path),
|
|
232
|
+
"screen_width": screen_width,
|
|
233
|
+
"screen_height": screen_height,
|
|
234
|
+
"original_img_width": original_img_width, # 截图原始宽度
|
|
235
|
+
"original_img_height": original_img_height, # 截图原始高度
|
|
236
|
+
"image_width": image_width, # 压缩后宽度(AI 看到的)
|
|
237
|
+
"image_height": image_height, # 压缩后高度(AI 看到的)
|
|
238
|
+
"original_size": f"{original_size/1024:.1f}KB",
|
|
239
|
+
"compressed_size": f"{compressed_size/1024:.1f}KB",
|
|
240
|
+
"saved_percent": f"{saved_percent:.0f}%",
|
|
241
|
+
"message": f"📸 截图已保存: {final_path}\n"
|
|
242
|
+
f"📐 原始尺寸: {original_img_width}x{original_img_height} → 压缩后: {image_width}x{image_height}\n"
|
|
243
|
+
f"📦 已压缩: {original_size/1024:.0f}KB → {compressed_size/1024:.0f}KB (省 {saved_percent:.0f}%)\n"
|
|
244
|
+
f"⚠️ 【坐标转换】AI 返回坐标后,请传入:\n"
|
|
245
|
+
f" image_width={image_width}, image_height={image_height},\n"
|
|
246
|
+
f" original_img_width={original_img_width}, original_img_height={original_img_height}"
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
# ========== 情况3:全屏不压缩截图 ==========
|
|
250
|
+
else:
|
|
251
|
+
if description:
|
|
252
|
+
safe_desc = re.sub(r'[^\w\s-]', '', description).strip().replace(' ', '_')
|
|
253
|
+
filename = f"screenshot_{platform}_{safe_desc}_{timestamp}.png"
|
|
254
|
+
else:
|
|
255
|
+
filename = f"screenshot_{platform}_{timestamp}.png"
|
|
256
|
+
|
|
257
|
+
final_path = self.screenshot_dir / filename
|
|
258
|
+
temp_path.rename(final_path)
|
|
259
|
+
|
|
260
|
+
# 不压缩时,用截图实际尺寸(可能和 screen_width 不同)
|
|
261
|
+
return {
|
|
262
|
+
"success": True,
|
|
263
|
+
"screenshot_path": str(final_path),
|
|
264
|
+
"screen_width": screen_width,
|
|
265
|
+
"screen_height": screen_height,
|
|
266
|
+
"original_img_width": img.width, # 截图实际尺寸
|
|
267
|
+
"original_img_height": img.height,
|
|
268
|
+
"image_width": img.width, # 未压缩,和原图一样
|
|
269
|
+
"image_height": img.height,
|
|
270
|
+
"file_size": f"{original_size/1024:.1f}KB",
|
|
271
|
+
"message": f"📸 截图已保存: {final_path}\n"
|
|
272
|
+
f"📐 截图尺寸: {img.width}x{img.height}\n"
|
|
273
|
+
f"📦 文件大小: {original_size/1024:.0f}KB(未压缩)\n"
|
|
274
|
+
f"💡 未压缩,坐标可直接使用"
|
|
275
|
+
}
|
|
276
|
+
except ImportError:
|
|
277
|
+
# 如果没有 PIL,回退到原始方式(不压缩)
|
|
278
|
+
return self._take_screenshot_no_compress(description)
|
|
279
|
+
except Exception as e:
|
|
280
|
+
return {"success": False, "message": f"❌ 截图失败: {e}"}
|
|
281
|
+
|
|
282
|
+
def take_screenshot_with_grid(self, grid_size: int = 100, show_popup_hints: bool = True) -> Dict:
|
|
283
|
+
"""截图并添加网格坐标标注(用于精确定位元素)
|
|
284
|
+
|
|
285
|
+
在截图上绘制网格线和坐标刻度,帮助快速定位元素位置。
|
|
286
|
+
如果检测到弹窗,会标注弹窗区域和可能的关闭按钮位置。
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
grid_size: 网格间距(像素),默认 100。建议值:50-200
|
|
290
|
+
show_popup_hints: 是否显示弹窗关闭按钮提示位置,默认 True
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
包含标注截图路径和弹窗信息的字典
|
|
294
|
+
"""
|
|
295
|
+
try:
|
|
296
|
+
from PIL import Image, ImageDraw, ImageFont
|
|
297
|
+
import re
|
|
298
|
+
|
|
299
|
+
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
300
|
+
platform = "ios" if self._is_ios() else "android"
|
|
301
|
+
|
|
302
|
+
# 第1步:截图
|
|
303
|
+
temp_filename = f"temp_grid_{timestamp}.png"
|
|
304
|
+
temp_path = self.screenshot_dir / temp_filename
|
|
305
|
+
|
|
306
|
+
screen_width, screen_height = 0, 0
|
|
307
|
+
if self._is_ios():
|
|
308
|
+
ios_client = self._get_ios_client()
|
|
309
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
310
|
+
ios_client.wda.screenshot(str(temp_path))
|
|
311
|
+
size = ios_client.wda.window_size()
|
|
312
|
+
screen_width, screen_height = size[0], size[1]
|
|
313
|
+
else:
|
|
314
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
315
|
+
else:
|
|
316
|
+
self.client.u2.screenshot(str(temp_path))
|
|
317
|
+
info = self.client.u2.info
|
|
318
|
+
screen_width = info.get('displayWidth', 720)
|
|
319
|
+
screen_height = info.get('displayHeight', 1280)
|
|
320
|
+
|
|
321
|
+
img = Image.open(temp_path)
|
|
322
|
+
draw = ImageDraw.Draw(img, 'RGBA')
|
|
323
|
+
|
|
324
|
+
# 尝试加载字体
|
|
325
|
+
try:
|
|
326
|
+
font = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 14)
|
|
327
|
+
font_small = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 11)
|
|
328
|
+
except:
|
|
329
|
+
font = ImageFont.load_default()
|
|
330
|
+
font_small = font
|
|
331
|
+
|
|
332
|
+
img_width, img_height = img.size
|
|
333
|
+
|
|
334
|
+
# 第2步:绘制网格线和坐标
|
|
335
|
+
grid_color = (255, 0, 0, 80) # 半透明红色
|
|
336
|
+
text_color = (255, 0, 0, 200) # 红色文字
|
|
337
|
+
|
|
338
|
+
# 绘制垂直网格线
|
|
339
|
+
for x in range(0, img_width, grid_size):
|
|
340
|
+
draw.line([(x, 0), (x, img_height)], fill=grid_color, width=1)
|
|
341
|
+
# 顶部标注 X 坐标
|
|
342
|
+
draw.text((x + 2, 2), str(x), fill=text_color, font=font_small)
|
|
343
|
+
|
|
344
|
+
# 绘制水平网格线
|
|
345
|
+
for y in range(0, img_height, grid_size):
|
|
346
|
+
draw.line([(0, y), (img_width, y)], fill=grid_color, width=1)
|
|
347
|
+
# 左侧标注 Y 坐标
|
|
348
|
+
draw.text((2, y + 2), str(y), fill=text_color, font=font_small)
|
|
349
|
+
|
|
350
|
+
# 第3步:检测弹窗并标注
|
|
351
|
+
popup_info = None
|
|
352
|
+
close_positions = []
|
|
353
|
+
|
|
354
|
+
if show_popup_hints and not self._is_ios():
|
|
355
|
+
try:
|
|
356
|
+
import xml.etree.ElementTree as ET
|
|
357
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
358
|
+
root = ET.fromstring(xml_string)
|
|
359
|
+
|
|
360
|
+
# 检测弹窗区域
|
|
361
|
+
popup_bounds = None
|
|
362
|
+
for elem in root.iter():
|
|
363
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
364
|
+
class_name = elem.attrib.get('class', '')
|
|
365
|
+
|
|
366
|
+
if not bounds_str:
|
|
367
|
+
continue
|
|
368
|
+
|
|
369
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
370
|
+
if not match:
|
|
371
|
+
continue
|
|
372
|
+
|
|
373
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
374
|
+
width = x2 - x1
|
|
375
|
+
height = y2 - y1
|
|
376
|
+
area = width * height
|
|
377
|
+
screen_area = screen_width * screen_height
|
|
378
|
+
|
|
379
|
+
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card'])
|
|
380
|
+
area_ratio = area / screen_area if screen_area > 0 else 0
|
|
381
|
+
is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
|
|
382
|
+
is_reasonable_size = 0.08 < area_ratio < 0.85
|
|
383
|
+
|
|
384
|
+
if is_container and is_not_fullscreen and is_reasonable_size and y1 > 50:
|
|
385
|
+
if popup_bounds is None or area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
386
|
+
popup_bounds = (x1, y1, x2, y2)
|
|
387
|
+
|
|
388
|
+
if popup_bounds:
|
|
389
|
+
px1, py1, px2, py2 = popup_bounds
|
|
390
|
+
popup_width = px2 - px1
|
|
391
|
+
popup_height = py2 - py1
|
|
392
|
+
|
|
393
|
+
# 绘制弹窗边框(蓝色)
|
|
394
|
+
draw.rectangle([px1, py1, px2, py2], outline=(0, 100, 255, 200), width=3)
|
|
395
|
+
draw.text((px1 + 5, py1 + 5), f"弹窗区域", fill=(0, 100, 255), font=font)
|
|
396
|
+
|
|
397
|
+
# 计算可能的 X 按钮位置(基于弹窗尺寸动态计算,适配不同分辨率)
|
|
398
|
+
offset_x = max(25, int(popup_width * 0.05)) # 宽度的5%,最小25px
|
|
399
|
+
offset_y = max(25, int(popup_height * 0.04)) # 高度的4%,最小25px
|
|
400
|
+
outer_offset = max(15, int(popup_width * 0.025)) # 外部偏移
|
|
401
|
+
|
|
402
|
+
close_positions = [
|
|
403
|
+
{"name": "右上角内", "x": px2 - offset_x, "y": py1 + offset_y, "priority": 1},
|
|
404
|
+
{"name": "右上角外", "x": px2 + outer_offset, "y": py1 - outer_offset, "priority": 2},
|
|
405
|
+
{"name": "正上方", "x": (px1 + px2) // 2, "y": py1 - offset_y, "priority": 3},
|
|
406
|
+
{"name": "底部下方", "x": (px1 + px2) // 2, "y": py2 + offset_y, "priority": 4},
|
|
407
|
+
]
|
|
408
|
+
|
|
409
|
+
# 绘制可能的 X 按钮位置(绿色圆圈 + 数字)
|
|
410
|
+
for i, pos in enumerate(close_positions):
|
|
411
|
+
cx, cy = pos["x"], pos["y"]
|
|
412
|
+
if 0 <= cx <= img_width and 0 <= cy <= img_height:
|
|
413
|
+
# 绿色圆圈
|
|
414
|
+
draw.ellipse([cx-15, cy-15, cx+15, cy+15],
|
|
415
|
+
outline=(0, 255, 0, 200), width=2)
|
|
416
|
+
# 数字标注
|
|
417
|
+
draw.text((cx-5, cy-8), str(i+1), fill=(0, 255, 0), font=font)
|
|
418
|
+
# 坐标标注
|
|
419
|
+
draw.text((cx+18, cy-8), f"({cx},{cy})", fill=(0, 255, 0), font=font_small)
|
|
420
|
+
|
|
421
|
+
popup_info = {
|
|
422
|
+
"bounds": f"[{px1},{py1}][{px2},{py2}]",
|
|
423
|
+
"width": px2 - px1,
|
|
424
|
+
"height": py2 - py1,
|
|
425
|
+
"close_positions": close_positions
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
except Exception as e:
|
|
429
|
+
pass # 弹窗检测失败不影响主功能
|
|
430
|
+
|
|
431
|
+
# 第4步:保存标注后的截图
|
|
432
|
+
filename = f"screenshot_{platform}_grid_{timestamp}.jpg"
|
|
433
|
+
final_path = self.screenshot_dir / filename
|
|
434
|
+
|
|
435
|
+
# 转换为 RGB 并保存
|
|
436
|
+
if img.mode in ('RGBA', 'LA', 'P'):
|
|
437
|
+
background = Image.new('RGB', img.size, (255, 255, 255))
|
|
438
|
+
if img.mode == 'P':
|
|
439
|
+
img = img.convert('RGBA')
|
|
440
|
+
background.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None)
|
|
441
|
+
img = background
|
|
442
|
+
elif img.mode != 'RGB':
|
|
443
|
+
img = img.convert("RGB")
|
|
444
|
+
|
|
445
|
+
img.save(str(final_path), "JPEG", quality=85)
|
|
446
|
+
temp_path.unlink()
|
|
447
|
+
|
|
448
|
+
result = {
|
|
449
|
+
"success": True,
|
|
450
|
+
"screenshot_path": str(final_path),
|
|
451
|
+
"screen_width": screen_width,
|
|
452
|
+
"screen_height": screen_height,
|
|
453
|
+
"image_width": img_width,
|
|
454
|
+
"image_height": img_height,
|
|
455
|
+
"grid_size": grid_size,
|
|
456
|
+
"message": f"📸 网格截图已保存: {final_path}\n"
|
|
457
|
+
f"📐 尺寸: {img_width}x{img_height}\n"
|
|
458
|
+
f"📏 网格间距: {grid_size}px"
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
if popup_info:
|
|
462
|
+
result["popup_detected"] = True
|
|
463
|
+
result["popup_bounds"] = popup_info["bounds"]
|
|
464
|
+
result["close_button_hints"] = close_positions
|
|
465
|
+
result["message"] += f"\n🎯 检测到弹窗: {popup_info['bounds']}"
|
|
466
|
+
result["message"] += f"\n💡 可能的关闭按钮位置(绿色圆圈标注):"
|
|
467
|
+
for pos in close_positions:
|
|
468
|
+
result["message"] += f"\n {pos['priority']}. {pos['name']}: ({pos['x']}, {pos['y']})"
|
|
469
|
+
else:
|
|
470
|
+
result["popup_detected"] = False
|
|
471
|
+
|
|
472
|
+
return result
|
|
473
|
+
|
|
474
|
+
except ImportError:
|
|
475
|
+
return {"success": False, "message": "❌ 需要安装 Pillow: pip install Pillow"}
|
|
476
|
+
except Exception as e:
|
|
477
|
+
return {"success": False, "message": f"❌ 网格截图失败: {e}"}
|
|
478
|
+
|
|
479
|
+
def take_screenshot_with_som(self) -> Dict:
|
|
480
|
+
"""Set-of-Mark 截图:给每个可点击元素标上数字(超级好用!)
|
|
481
|
+
|
|
482
|
+
在截图上给每个可点击元素画框并标上数字编号。
|
|
483
|
+
AI 看图后直接说"点击 3 号",然后调用 click_by_som(3) 即可。
|
|
484
|
+
|
|
485
|
+
Returns:
|
|
486
|
+
包含标注截图和元素列表的字典
|
|
487
|
+
"""
|
|
488
|
+
try:
|
|
489
|
+
from PIL import Image, ImageDraw, ImageFont
|
|
490
|
+
import re
|
|
491
|
+
|
|
492
|
+
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
493
|
+
platform = "ios" if self._is_ios() else "android"
|
|
494
|
+
|
|
495
|
+
# 第1步:截图
|
|
496
|
+
temp_filename = f"temp_som_{timestamp}.png"
|
|
497
|
+
temp_path = self.screenshot_dir / temp_filename
|
|
498
|
+
|
|
499
|
+
screen_width, screen_height = 0, 0
|
|
500
|
+
if self._is_ios():
|
|
501
|
+
ios_client = self._get_ios_client()
|
|
502
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
503
|
+
ios_client.wda.screenshot(str(temp_path))
|
|
504
|
+
size = ios_client.wda.window_size()
|
|
505
|
+
screen_width, screen_height = size[0], size[1]
|
|
506
|
+
else:
|
|
507
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
508
|
+
else:
|
|
509
|
+
self.client.u2.screenshot(str(temp_path))
|
|
510
|
+
info = self.client.u2.info
|
|
511
|
+
screen_width = info.get('displayWidth', 720)
|
|
512
|
+
screen_height = info.get('displayHeight', 1280)
|
|
513
|
+
|
|
514
|
+
img = Image.open(temp_path)
|
|
515
|
+
draw = ImageDraw.Draw(img, 'RGBA')
|
|
516
|
+
img_width, img_height = img.size
|
|
517
|
+
|
|
518
|
+
# 尝试加载字体
|
|
519
|
+
try:
|
|
520
|
+
font = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 16)
|
|
521
|
+
font_small = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 12)
|
|
522
|
+
except:
|
|
523
|
+
font = ImageFont.load_default()
|
|
524
|
+
font_small = font
|
|
525
|
+
|
|
526
|
+
# 第2步:获取所有可点击元素
|
|
527
|
+
elements = []
|
|
528
|
+
if self._is_ios():
|
|
529
|
+
# iOS 暂不支持
|
|
530
|
+
pass
|
|
531
|
+
else:
|
|
532
|
+
try:
|
|
533
|
+
import xml.etree.ElementTree as ET
|
|
534
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
535
|
+
root = ET.fromstring(xml_string)
|
|
536
|
+
|
|
537
|
+
for elem in root.iter():
|
|
538
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
539
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
540
|
+
text = elem.attrib.get('text', '')
|
|
541
|
+
content_desc = elem.attrib.get('content-desc', '')
|
|
542
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
543
|
+
class_name = elem.attrib.get('class', '')
|
|
544
|
+
|
|
545
|
+
if not clickable or not bounds_str:
|
|
546
|
+
continue
|
|
547
|
+
|
|
548
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
549
|
+
if not match:
|
|
550
|
+
continue
|
|
551
|
+
|
|
552
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
553
|
+
width = x2 - x1
|
|
554
|
+
height = y2 - y1
|
|
555
|
+
|
|
556
|
+
# 过滤太小或太大的元素
|
|
557
|
+
if width < 20 or height < 20:
|
|
558
|
+
continue
|
|
559
|
+
if width >= screen_width * 0.98 and height >= screen_height * 0.5:
|
|
560
|
+
continue # 全屏或大面积容器
|
|
561
|
+
|
|
562
|
+
center_x = (x1 + x2) // 2
|
|
563
|
+
center_y = (y1 + y2) // 2
|
|
564
|
+
|
|
565
|
+
# 生成描述
|
|
566
|
+
desc = text or content_desc or resource_id.split('/')[-1] if resource_id else class_name.split('.')[-1]
|
|
567
|
+
if len(desc) > 20:
|
|
568
|
+
desc = desc[:17] + "..."
|
|
569
|
+
|
|
570
|
+
elements.append({
|
|
571
|
+
'bounds': (x1, y1, x2, y2),
|
|
572
|
+
'center': (center_x, center_y),
|
|
573
|
+
'text': text,
|
|
574
|
+
'desc': desc,
|
|
575
|
+
'resource_id': resource_id
|
|
576
|
+
})
|
|
577
|
+
except Exception as e:
|
|
578
|
+
pass
|
|
579
|
+
|
|
580
|
+
# 第3步:在截图上标注元素
|
|
581
|
+
# 颜色列表(循环使用)
|
|
582
|
+
colors = [
|
|
583
|
+
(255, 0, 0), # 红
|
|
584
|
+
(0, 255, 0), # 绿
|
|
585
|
+
(0, 100, 255), # 蓝
|
|
586
|
+
(255, 165, 0), # 橙
|
|
587
|
+
(255, 0, 255), # 紫
|
|
588
|
+
(0, 255, 255), # 青
|
|
589
|
+
]
|
|
590
|
+
|
|
591
|
+
som_elements = [] # 保存标注信息,供 click_by_som 使用
|
|
592
|
+
|
|
593
|
+
for i, elem in enumerate(elements):
|
|
594
|
+
x1, y1, x2, y2 = elem['bounds']
|
|
595
|
+
cx, cy = elem['center']
|
|
596
|
+
color = colors[i % len(colors)]
|
|
597
|
+
|
|
598
|
+
# 画边框
|
|
599
|
+
draw.rectangle([x1, y1, x2, y2], outline=color + (200,), width=2)
|
|
600
|
+
|
|
601
|
+
# 画编号标签背景
|
|
602
|
+
label = str(i + 1)
|
|
603
|
+
label_w, label_h = 20, 18
|
|
604
|
+
label_x = x1
|
|
605
|
+
label_y = max(0, y1 - label_h - 2)
|
|
606
|
+
draw.rectangle([label_x, label_y, label_x + label_w, label_y + label_h],
|
|
607
|
+
fill=color + (220,))
|
|
608
|
+
|
|
609
|
+
# 画编号文字
|
|
610
|
+
draw.text((label_x + 4, label_y + 1), label, fill=(255, 255, 255), font=font_small)
|
|
611
|
+
|
|
612
|
+
som_elements.append({
|
|
613
|
+
'index': i + 1,
|
|
614
|
+
'center': (cx, cy),
|
|
615
|
+
'bounds': f"[{x1},{y1}][{x2},{y2}]",
|
|
616
|
+
'desc': elem['desc']
|
|
617
|
+
})
|
|
618
|
+
|
|
619
|
+
# 第3.5步:检测弹窗区域(用于标注)
|
|
620
|
+
popup_bounds = None
|
|
621
|
+
|
|
622
|
+
if not self._is_ios():
|
|
623
|
+
try:
|
|
624
|
+
# 检测弹窗区域
|
|
625
|
+
for elem in root.iter():
|
|
626
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
627
|
+
class_name = elem.attrib.get('class', '')
|
|
628
|
+
|
|
629
|
+
if not bounds_str:
|
|
630
|
+
continue
|
|
631
|
+
|
|
632
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
633
|
+
if not match:
|
|
634
|
+
continue
|
|
635
|
+
|
|
636
|
+
px1, py1, px2, py2 = map(int, match.groups())
|
|
637
|
+
p_width = px2 - px1
|
|
638
|
+
p_height = py2 - py1
|
|
639
|
+
p_area = p_width * p_height
|
|
640
|
+
screen_area = screen_width * screen_height
|
|
641
|
+
|
|
642
|
+
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Frame'])
|
|
643
|
+
area_ratio = p_area / screen_area if screen_area > 0 else 0
|
|
644
|
+
is_not_fullscreen = (p_width < screen_width * 0.99 or p_height < screen_height * 0.95)
|
|
645
|
+
# 放宽面积范围:5% - 95%
|
|
646
|
+
is_reasonable_size = 0.05 < area_ratio < 0.95
|
|
647
|
+
|
|
648
|
+
if is_container and is_not_fullscreen and is_reasonable_size and py1 > 30:
|
|
649
|
+
if popup_bounds is None or p_area > (popup_bounds[2] - popup_bounds[0]) * (popup_bounds[3] - popup_bounds[1]):
|
|
650
|
+
popup_bounds = (px1, py1, px2, py2)
|
|
651
|
+
|
|
652
|
+
# 如果检测到弹窗,标注弹窗边界(不再猜测X按钮位置)
|
|
653
|
+
if popup_bounds:
|
|
654
|
+
px1, py1, px2, py2 = popup_bounds
|
|
655
|
+
|
|
656
|
+
# 只画弹窗边框(蓝色),不再猜测X按钮位置
|
|
657
|
+
draw.rectangle([px1, py1, px2, py2], outline=(0, 150, 255, 180), width=3)
|
|
658
|
+
|
|
659
|
+
# 在弹窗边框上标注提示文字
|
|
660
|
+
try:
|
|
661
|
+
draw.text((px1+5, py1-25), "弹窗区域", fill=(0, 150, 255), font=font_small)
|
|
662
|
+
except:
|
|
663
|
+
pass
|
|
664
|
+
|
|
665
|
+
except Exception as e:
|
|
666
|
+
pass # 弹窗检测失败不影响主功能
|
|
667
|
+
|
|
668
|
+
# 保存到实例变量,供 click_by_som 使用
|
|
669
|
+
self._som_elements = som_elements
|
|
670
|
+
|
|
671
|
+
# 第4步:保存标注后的截图
|
|
672
|
+
filename = f"screenshot_{platform}_som_{timestamp}.jpg"
|
|
673
|
+
final_path = self.screenshot_dir / filename
|
|
674
|
+
|
|
675
|
+
if img.mode in ('RGBA', 'LA', 'P'):
|
|
676
|
+
background = Image.new('RGB', img.size, (255, 255, 255))
|
|
677
|
+
if img.mode == 'P':
|
|
678
|
+
img = img.convert('RGBA')
|
|
679
|
+
background.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None)
|
|
680
|
+
img = background
|
|
681
|
+
elif img.mode != 'RGB':
|
|
682
|
+
img = img.convert("RGB")
|
|
683
|
+
|
|
684
|
+
img.save(str(final_path), "JPEG", quality=85)
|
|
685
|
+
temp_path.unlink()
|
|
686
|
+
|
|
687
|
+
# 构建元素列表文字
|
|
688
|
+
elements_text = "\n".join([
|
|
689
|
+
f" [{e['index']}] {e['desc']} → ({e['center'][0]}, {e['center'][1]})"
|
|
690
|
+
for e in som_elements[:15] # 只显示前15个
|
|
691
|
+
])
|
|
692
|
+
if len(som_elements) > 15:
|
|
693
|
+
elements_text += f"\n ... 还有 {len(som_elements) - 15} 个元素"
|
|
694
|
+
|
|
695
|
+
# 构建弹窗提示文字
|
|
696
|
+
hints_text = ""
|
|
697
|
+
if popup_bounds:
|
|
698
|
+
hints_text = f"\n🎯 检测到弹窗区域(蓝色边框)\n"
|
|
699
|
+
hints_text += f" 如需关闭弹窗,请观察图片中的 X 按钮位置\n"
|
|
700
|
+
hints_text += f" 然后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
701
|
+
|
|
702
|
+
return {
|
|
703
|
+
"success": True,
|
|
704
|
+
"screenshot_path": str(final_path),
|
|
705
|
+
"screen_width": screen_width,
|
|
706
|
+
"screen_height": screen_height,
|
|
707
|
+
"image_width": img_width,
|
|
708
|
+
"image_height": img_height,
|
|
709
|
+
"element_count": len(som_elements),
|
|
710
|
+
"elements": som_elements,
|
|
711
|
+
"popup_detected": popup_bounds is not None,
|
|
712
|
+
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
|
|
713
|
+
"message": f"📸 SoM 截图已保存: {final_path}\n"
|
|
714
|
+
f"🏷️ 已标注 {len(som_elements)} 个可点击元素\n"
|
|
715
|
+
f"📋 元素列表:\n{elements_text}{hints_text}\n\n"
|
|
716
|
+
f"💡 使用方法:\n"
|
|
717
|
+
f" - 点击标注元素:mobile_click_by_som(编号)\n"
|
|
718
|
+
f" - 点击任意位置:mobile_click_by_percent(x%, y%)"
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
except ImportError:
|
|
722
|
+
return {"success": False, "message": "❌ 需要安装 Pillow: pip install Pillow"}
|
|
723
|
+
except Exception as e:
|
|
724
|
+
return {"success": False, "message": f"❌ SoM 截图失败: {e}"}
|
|
725
|
+
|
|
726
|
+
def click_by_som(self, index: int) -> Dict:
|
|
727
|
+
"""根据 SoM 编号点击元素
|
|
728
|
+
|
|
729
|
+
配合 take_screenshot_with_som 使用。
|
|
730
|
+
看图后直接说"点击 3 号",调用此函数即可。
|
|
731
|
+
|
|
732
|
+
Args:
|
|
733
|
+
index: 元素编号(从 1 开始)
|
|
734
|
+
|
|
735
|
+
Returns:
|
|
736
|
+
点击结果
|
|
737
|
+
"""
|
|
738
|
+
try:
|
|
739
|
+
if not hasattr(self, '_som_elements') or not self._som_elements:
|
|
740
|
+
return {
|
|
741
|
+
"success": False,
|
|
742
|
+
"message": "❌ 请先调用 mobile_screenshot_with_som 获取元素列表"
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
# 查找对应编号的元素
|
|
746
|
+
target = None
|
|
747
|
+
for elem in self._som_elements:
|
|
748
|
+
if elem['index'] == index:
|
|
749
|
+
target = elem
|
|
750
|
+
break
|
|
751
|
+
|
|
752
|
+
if not target:
|
|
753
|
+
return {
|
|
754
|
+
"success": False,
|
|
755
|
+
"message": f"❌ 未找到编号 {index} 的元素,有效范围: 1-{len(self._som_elements)}"
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
# 点击
|
|
759
|
+
cx, cy = target['center']
|
|
760
|
+
if self._is_ios():
|
|
761
|
+
ios_client = self._get_ios_client()
|
|
762
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
763
|
+
ios_client.wda.click(cx, cy)
|
|
764
|
+
else:
|
|
765
|
+
self.client.u2.click(cx, cy)
|
|
766
|
+
|
|
767
|
+
time.sleep(0.3)
|
|
768
|
+
|
|
769
|
+
return {
|
|
770
|
+
"success": True,
|
|
771
|
+
"message": f"✅ 已点击 [{index}] {target['desc']} → ({cx}, {cy})\n💡 建议:再次截图确认操作是否成功",
|
|
772
|
+
"clicked": {
|
|
773
|
+
"index": index,
|
|
774
|
+
"desc": target['desc'],
|
|
775
|
+
"coords": (cx, cy),
|
|
776
|
+
"bounds": target['bounds']
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
except Exception as e:
|
|
781
|
+
return {"success": False, "message": f"❌ 点击失败: {e}\n💡 如果页面已变化,请重新调用 mobile_screenshot_with_som 刷新元素列表"}
|
|
782
|
+
|
|
783
|
+
def _take_screenshot_no_compress(self, description: str = "") -> Dict:
|
|
784
|
+
"""截图(不压缩,PIL 不可用时的备用方案)"""
|
|
785
|
+
try:
|
|
786
|
+
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
787
|
+
platform = "ios" if self._is_ios() else "android"
|
|
788
|
+
|
|
789
|
+
if description:
|
|
790
|
+
safe_desc = re.sub(r'[^\w\s-]', '', description).strip().replace(' ', '_')
|
|
791
|
+
filename = f"screenshot_{platform}_{safe_desc}_{timestamp}.png"
|
|
792
|
+
else:
|
|
793
|
+
filename = f"screenshot_{platform}_{timestamp}.png"
|
|
794
|
+
|
|
795
|
+
screenshot_path = self.screenshot_dir / filename
|
|
796
|
+
|
|
797
|
+
width, height = 0, 0
|
|
798
|
+
if self._is_ios():
|
|
799
|
+
ios_client = self._get_ios_client()
|
|
800
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
801
|
+
ios_client.wda.screenshot(str(screenshot_path))
|
|
802
|
+
size = ios_client.wda.window_size()
|
|
803
|
+
width, height = size[0], size[1]
|
|
804
|
+
else:
|
|
805
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
806
|
+
else:
|
|
807
|
+
self.client.u2.screenshot(str(screenshot_path))
|
|
808
|
+
info = self.client.u2.info
|
|
809
|
+
width = info.get('displayWidth', 0)
|
|
810
|
+
height = info.get('displayHeight', 0)
|
|
811
|
+
|
|
812
|
+
# 不压缩时,图片尺寸 = 屏幕尺寸
|
|
813
|
+
return {
|
|
814
|
+
"success": True,
|
|
815
|
+
"screenshot_path": str(screenshot_path),
|
|
816
|
+
"screen_width": width,
|
|
817
|
+
"screen_height": height,
|
|
818
|
+
"image_width": width,
|
|
819
|
+
"image_height": height,
|
|
820
|
+
"message": f"📸 截图已保存: {screenshot_path}\n"
|
|
821
|
+
f"📐 屏幕尺寸: {width}x{height}\n"
|
|
822
|
+
f"⚠️ 未压缩(PIL 未安装),建议安装: pip install Pillow"
|
|
823
|
+
}
|
|
824
|
+
except Exception as e:
|
|
825
|
+
return {"success": False, "message": f"❌ 截图失败: {e}"}
|
|
826
|
+
|
|
827
|
+
def get_screen_size(self) -> Dict:
|
|
828
|
+
"""获取屏幕尺寸"""
|
|
829
|
+
try:
|
|
830
|
+
if self._is_ios():
|
|
831
|
+
ios_client = self._get_ios_client()
|
|
832
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
833
|
+
size = ios_client.wda.window_size()
|
|
834
|
+
return {
|
|
835
|
+
"success": True,
|
|
836
|
+
"width": size[0],
|
|
837
|
+
"height": size[1],
|
|
838
|
+
"size": f"{size[0]}x{size[1]}"
|
|
839
|
+
}
|
|
840
|
+
else:
|
|
841
|
+
info = self.client.u2.info
|
|
842
|
+
width = info.get('displayWidth', 0)
|
|
843
|
+
height = info.get('displayHeight', 0)
|
|
844
|
+
return {
|
|
845
|
+
"success": True,
|
|
846
|
+
"width": width,
|
|
847
|
+
"height": height,
|
|
848
|
+
"size": f"{width}x{height}"
|
|
849
|
+
}
|
|
850
|
+
except Exception as e:
|
|
851
|
+
return {"success": False, "message": f"❌ 获取屏幕尺寸失败: {e}"}
|
|
852
|
+
|
|
853
|
+
# ==================== 点击操作 ====================
|
|
854
|
+
|
|
855
|
+
def click_at_coords(self, x: int, y: int, image_width: int = 0, image_height: int = 0,
|
|
856
|
+
crop_offset_x: int = 0, crop_offset_y: int = 0,
|
|
857
|
+
original_img_width: int = 0, original_img_height: int = 0) -> Dict:
|
|
858
|
+
"""点击坐标(核心功能,支持自动坐标转换)
|
|
859
|
+
|
|
860
|
+
Args:
|
|
861
|
+
x: X 坐标(来自截图分析或屏幕坐标)
|
|
862
|
+
y: Y 坐标(来自截图分析或屏幕坐标)
|
|
863
|
+
image_width: 压缩后图片宽度(AI 看到的图片尺寸)
|
|
864
|
+
image_height: 压缩后图片高度(AI 看到的图片尺寸)
|
|
865
|
+
crop_offset_x: 局部截图的 X 偏移量(局部截图时传入)
|
|
866
|
+
crop_offset_y: 局部截图的 Y 偏移量(局部截图时传入)
|
|
867
|
+
original_img_width: 截图原始宽度(压缩前的尺寸,用于精确转换)
|
|
868
|
+
original_img_height: 截图原始高度(压缩前的尺寸,用于精确转换)
|
|
869
|
+
|
|
870
|
+
坐标转换说明:
|
|
871
|
+
1. 全屏压缩截图:AI 坐标 → 原图坐标(基于 image/original_img 比例)
|
|
872
|
+
2. 局部裁剪截图:AI 坐标 + 偏移量 = 屏幕坐标
|
|
873
|
+
"""
|
|
874
|
+
try:
|
|
875
|
+
# 获取屏幕尺寸
|
|
876
|
+
screen_width, screen_height = 0, 0
|
|
877
|
+
if self._is_ios():
|
|
878
|
+
ios_client = self._get_ios_client()
|
|
879
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
880
|
+
size = ios_client.wda.window_size()
|
|
881
|
+
screen_width, screen_height = size[0], size[1]
|
|
882
|
+
else:
|
|
883
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
884
|
+
else:
|
|
885
|
+
info = self.client.u2.info
|
|
886
|
+
screen_width = info.get('displayWidth', 0)
|
|
887
|
+
screen_height = info.get('displayHeight', 0)
|
|
888
|
+
|
|
889
|
+
# 🎯 坐标转换
|
|
890
|
+
original_x, original_y = x, y
|
|
891
|
+
converted = False
|
|
892
|
+
conversion_type = ""
|
|
893
|
+
|
|
894
|
+
# 情况1:局部裁剪截图 - 加上偏移量
|
|
895
|
+
if crop_offset_x > 0 or crop_offset_y > 0:
|
|
896
|
+
x = x + crop_offset_x
|
|
897
|
+
y = y + crop_offset_y
|
|
898
|
+
converted = True
|
|
899
|
+
conversion_type = "crop_offset"
|
|
900
|
+
# 情况2:全屏压缩截图 - 按比例转换到原图尺寸
|
|
901
|
+
elif image_width > 0 and image_height > 0:
|
|
902
|
+
# 优先使用 original_img_width/height(更精确)
|
|
903
|
+
# 如果没传,则用 screen_width/height(兼容旧版本)
|
|
904
|
+
target_width = original_img_width if original_img_width > 0 else screen_width
|
|
905
|
+
target_height = original_img_height if original_img_height > 0 else screen_height
|
|
906
|
+
|
|
907
|
+
if target_width > 0 and target_height > 0:
|
|
908
|
+
if image_width != target_width or image_height != target_height:
|
|
909
|
+
x = int(x * target_width / image_width)
|
|
910
|
+
y = int(y * target_height / image_height)
|
|
911
|
+
converted = True
|
|
912
|
+
conversion_type = "scale"
|
|
913
|
+
|
|
914
|
+
# 执行点击
|
|
915
|
+
if self._is_ios():
|
|
916
|
+
ios_client = self._get_ios_client()
|
|
917
|
+
ios_client.wda.click(x, y)
|
|
918
|
+
else:
|
|
919
|
+
self.client.u2.click(x, y)
|
|
920
|
+
|
|
921
|
+
time.sleep(0.3)
|
|
922
|
+
|
|
923
|
+
# 计算百分比坐标(用于跨设备兼容)
|
|
924
|
+
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
925
|
+
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
926
|
+
|
|
927
|
+
# 记录操作(包含屏幕尺寸和百分比,便于脚本生成时转换)
|
|
928
|
+
self._record_operation(
|
|
929
|
+
'click',
|
|
930
|
+
x=x,
|
|
931
|
+
y=y,
|
|
932
|
+
x_percent=x_percent,
|
|
933
|
+
y_percent=y_percent,
|
|
934
|
+
screen_width=screen_width,
|
|
935
|
+
screen_height=screen_height,
|
|
936
|
+
ref=f"coords_{x}_{y}"
|
|
937
|
+
)
|
|
938
|
+
|
|
939
|
+
if converted:
|
|
940
|
+
if conversion_type == "crop_offset":
|
|
941
|
+
return {
|
|
942
|
+
"success": True,
|
|
943
|
+
"message": f"✅ 点击成功: ({x}, {y})\n"
|
|
944
|
+
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
945
|
+
}
|
|
946
|
+
else:
|
|
947
|
+
return {
|
|
948
|
+
"success": True,
|
|
949
|
+
"message": f"✅ 点击成功: ({x}, {y})\n"
|
|
950
|
+
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
951
|
+
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
952
|
+
}
|
|
953
|
+
else:
|
|
954
|
+
return {
|
|
955
|
+
"success": True,
|
|
956
|
+
"message": f"✅ 点击成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%]"
|
|
957
|
+
}
|
|
958
|
+
except Exception as e:
|
|
959
|
+
return {"success": False, "message": f"❌ 点击失败: {e}"}
|
|
960
|
+
|
|
961
|
+
def click_by_percent(self, x_percent: float, y_percent: float) -> Dict:
|
|
962
|
+
"""通过百分比坐标点击(跨设备兼容)
|
|
963
|
+
|
|
964
|
+
百分比坐标原理:
|
|
965
|
+
- 屏幕左上角是 (0%, 0%),右下角是 (100%, 100%)
|
|
966
|
+
- 屏幕正中央是 (50%, 50%)
|
|
967
|
+
- 像素坐标 = 屏幕尺寸 × (百分比 / 100)
|
|
968
|
+
|
|
969
|
+
Args:
|
|
970
|
+
x_percent: X轴百分比 (0-100),0=最左,50=中间,100=最右
|
|
971
|
+
y_percent: Y轴百分比 (0-100),0=最上,50=中间,100=最下
|
|
972
|
+
|
|
973
|
+
示例:
|
|
974
|
+
click_by_percent(50, 50) # 点击屏幕正中央
|
|
975
|
+
click_by_percent(10, 5) # 点击左上角附近
|
|
976
|
+
click_by_percent(85, 90) # 点击右下角附近
|
|
977
|
+
|
|
978
|
+
优势:
|
|
979
|
+
- 同样的百分比在不同分辨率设备上都能点到相同相对位置
|
|
980
|
+
- 录制一次,多设备回放
|
|
981
|
+
"""
|
|
982
|
+
try:
|
|
983
|
+
# 第1步:获取屏幕尺寸
|
|
984
|
+
if self._is_ios():
|
|
985
|
+
ios_client = self._get_ios_client()
|
|
986
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
987
|
+
size = ios_client.wda.window_size()
|
|
988
|
+
width, height = size[0], size[1]
|
|
989
|
+
else:
|
|
990
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
991
|
+
else:
|
|
992
|
+
info = self.client.u2.info
|
|
993
|
+
width = info.get('displayWidth', 0)
|
|
994
|
+
height = info.get('displayHeight', 0)
|
|
995
|
+
|
|
996
|
+
if width == 0 or height == 0:
|
|
997
|
+
return {"success": False, "message": "❌ 无法获取屏幕尺寸"}
|
|
998
|
+
|
|
999
|
+
# 第2步:百分比转像素坐标
|
|
1000
|
+
# 公式:像素 = 屏幕尺寸 × (百分比 / 100)
|
|
1001
|
+
x = int(width * x_percent / 100)
|
|
1002
|
+
y = int(height * y_percent / 100)
|
|
1003
|
+
|
|
1004
|
+
# 第3步:执行点击
|
|
1005
|
+
if self._is_ios():
|
|
1006
|
+
ios_client.wda.click(x, y)
|
|
1007
|
+
else:
|
|
1008
|
+
self.client.u2.click(x, y)
|
|
1009
|
+
|
|
1010
|
+
time.sleep(0.3)
|
|
1011
|
+
|
|
1012
|
+
# 第4步:记录操作(同时记录百分比和像素)
|
|
1013
|
+
self._record_operation(
|
|
1014
|
+
'click',
|
|
1015
|
+
x=x,
|
|
1016
|
+
y=y,
|
|
1017
|
+
x_percent=x_percent,
|
|
1018
|
+
y_percent=y_percent,
|
|
1019
|
+
screen_width=width,
|
|
1020
|
+
screen_height=height,
|
|
1021
|
+
ref=f"percent_{x_percent}_{y_percent}"
|
|
1022
|
+
)
|
|
1023
|
+
|
|
1024
|
+
return {
|
|
1025
|
+
"success": True,
|
|
1026
|
+
"message": f"✅ 百分比点击成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y})",
|
|
1027
|
+
"screen_size": {"width": width, "height": height},
|
|
1028
|
+
"percent": {"x": x_percent, "y": y_percent},
|
|
1029
|
+
"pixel": {"x": x, "y": y}
|
|
1030
|
+
}
|
|
1031
|
+
except Exception as e:
|
|
1032
|
+
return {"success": False, "message": f"❌ 百分比点击失败: {e}"}
|
|
1033
|
+
|
|
1034
|
+
def click_by_text(self, text: str, timeout: float = 3.0) -> Dict:
|
|
1035
|
+
"""通过文本点击 - 先查 XML 树,再精准匹配"""
|
|
1036
|
+
try:
|
|
1037
|
+
if self._is_ios():
|
|
1038
|
+
ios_client = self._get_ios_client()
|
|
1039
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1040
|
+
elem = ios_client.wda(name=text)
|
|
1041
|
+
if not elem.exists:
|
|
1042
|
+
elem = ios_client.wda(label=text)
|
|
1043
|
+
if elem.exists:
|
|
1044
|
+
elem.click()
|
|
1045
|
+
time.sleep(0.3)
|
|
1046
|
+
self._record_operation('click', element=text, ref=text)
|
|
1047
|
+
return {"success": True, "message": f"✅ 点击成功: '{text}'"}
|
|
1048
|
+
return {"success": False, "message": f"❌ 文本不存在: {text}"}
|
|
1049
|
+
else:
|
|
1050
|
+
# 🔍 先查 XML 树,找到元素及其属性
|
|
1051
|
+
found_elem = self._find_element_in_tree(text)
|
|
1052
|
+
|
|
1053
|
+
if found_elem:
|
|
1054
|
+
attr_type = found_elem['attr_type']
|
|
1055
|
+
attr_value = found_elem['attr_value']
|
|
1056
|
+
bounds = found_elem.get('bounds')
|
|
1057
|
+
|
|
1058
|
+
# 根据找到的属性类型,使用对应的选择器
|
|
1059
|
+
if attr_type == 'text':
|
|
1060
|
+
elem = self.client.u2(text=attr_value)
|
|
1061
|
+
elif attr_type == 'textContains':
|
|
1062
|
+
elem = self.client.u2(textContains=attr_value)
|
|
1063
|
+
elif attr_type == 'description':
|
|
1064
|
+
elem = self.client.u2(description=attr_value)
|
|
1065
|
+
elif attr_type == 'descriptionContains':
|
|
1066
|
+
elem = self.client.u2(descriptionContains=attr_value)
|
|
1067
|
+
else:
|
|
1068
|
+
elem = None
|
|
1069
|
+
|
|
1070
|
+
if elem and elem.exists(timeout=1):
|
|
1071
|
+
elem.click()
|
|
1072
|
+
time.sleep(0.3)
|
|
1073
|
+
self._record_operation('click', element=text, ref=f"{attr_type}:{attr_value}")
|
|
1074
|
+
return {"success": True, "message": f"✅ 点击成功({attr_type}): '{text}'"}
|
|
1075
|
+
|
|
1076
|
+
# 如果选择器失败,用坐标兜底
|
|
1077
|
+
if bounds:
|
|
1078
|
+
x = (bounds[0] + bounds[2]) // 2
|
|
1079
|
+
y = (bounds[1] + bounds[3]) // 2
|
|
1080
|
+
self.client.u2.click(x, y)
|
|
1081
|
+
time.sleep(0.3)
|
|
1082
|
+
self._record_operation('click', element=text, x=x, y=y, ref=f"coords:{x},{y}")
|
|
1083
|
+
return {"success": True, "message": f"✅ 点击成功(坐标兜底): '{text}' @ ({x},{y})"}
|
|
1084
|
+
|
|
1085
|
+
return {"success": False, "message": f"❌ 文本不存在: {text}"}
|
|
1086
|
+
except Exception as e:
|
|
1087
|
+
return {"success": False, "message": f"❌ 点击失败: {e}"}
|
|
1088
|
+
|
|
1089
|
+
def _find_element_in_tree(self, text: str) -> Optional[Dict]:
|
|
1090
|
+
"""在 XML 树中查找包含指定文本的元素"""
|
|
1091
|
+
try:
|
|
1092
|
+
xml = self.client.u2.dump_hierarchy(compressed=False)
|
|
1093
|
+
import xml.etree.ElementTree as ET
|
|
1094
|
+
root = ET.fromstring(xml)
|
|
1095
|
+
|
|
1096
|
+
for elem in root.iter():
|
|
1097
|
+
elem_text = elem.attrib.get('text', '')
|
|
1098
|
+
elem_desc = elem.attrib.get('content-desc', '')
|
|
1099
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
1100
|
+
|
|
1101
|
+
# 解析 bounds
|
|
1102
|
+
bounds = None
|
|
1103
|
+
if bounds_str:
|
|
1104
|
+
import re
|
|
1105
|
+
match = re.findall(r'\d+', bounds_str)
|
|
1106
|
+
if len(match) == 4:
|
|
1107
|
+
bounds = [int(x) for x in match]
|
|
1108
|
+
|
|
1109
|
+
# 精确匹配 text
|
|
1110
|
+
if elem_text == text:
|
|
1111
|
+
return {'attr_type': 'text', 'attr_value': text, 'bounds': bounds}
|
|
1112
|
+
|
|
1113
|
+
# 精确匹配 content-desc
|
|
1114
|
+
if elem_desc == text:
|
|
1115
|
+
return {'attr_type': 'description', 'attr_value': text, 'bounds': bounds}
|
|
1116
|
+
|
|
1117
|
+
# 模糊匹配 text
|
|
1118
|
+
if text in elem_text:
|
|
1119
|
+
return {'attr_type': 'textContains', 'attr_value': text, 'bounds': bounds}
|
|
1120
|
+
|
|
1121
|
+
# 模糊匹配 content-desc
|
|
1122
|
+
if text in elem_desc:
|
|
1123
|
+
return {'attr_type': 'descriptionContains', 'attr_value': text, 'bounds': bounds}
|
|
1124
|
+
|
|
1125
|
+
return None
|
|
1126
|
+
except Exception:
|
|
1127
|
+
return None
|
|
1128
|
+
|
|
1129
|
+
def click_by_id(self, resource_id: str, index: int = 0) -> Dict:
|
|
1130
|
+
"""通过 resource-id 点击(支持点击第 N 个元素)
|
|
1131
|
+
|
|
1132
|
+
Args:
|
|
1133
|
+
resource_id: 元素的 resource-id
|
|
1134
|
+
index: 第几个元素(从 0 开始),默认 0 表示第一个
|
|
1135
|
+
"""
|
|
1136
|
+
try:
|
|
1137
|
+
index_desc = f"[{index}]" if index > 0 else ""
|
|
1138
|
+
|
|
1139
|
+
if self._is_ios():
|
|
1140
|
+
ios_client = self._get_ios_client()
|
|
1141
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1142
|
+
elem = ios_client.wda(id=resource_id)
|
|
1143
|
+
if not elem.exists:
|
|
1144
|
+
elem = ios_client.wda(name=resource_id)
|
|
1145
|
+
if elem.exists:
|
|
1146
|
+
# 获取所有匹配的元素
|
|
1147
|
+
elements = elem.find_elements()
|
|
1148
|
+
if index < len(elements):
|
|
1149
|
+
elements[index].click()
|
|
1150
|
+
time.sleep(0.3)
|
|
1151
|
+
self._record_operation('click', element=f"{resource_id}{index_desc}", ref=resource_id, index=index)
|
|
1152
|
+
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}"}
|
|
1153
|
+
else:
|
|
1154
|
+
return {"success": False, "message": f"❌ 索引超出范围: 找到 {len(elements)} 个元素,但请求索引 {index}"}
|
|
1155
|
+
return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
|
|
1156
|
+
else:
|
|
1157
|
+
elem = self.client.u2(resourceId=resource_id)
|
|
1158
|
+
if elem.exists(timeout=0.5):
|
|
1159
|
+
# 获取匹配元素数量
|
|
1160
|
+
count = elem.count
|
|
1161
|
+
if index < count:
|
|
1162
|
+
elem[index].click()
|
|
1163
|
+
time.sleep(0.3)
|
|
1164
|
+
self._record_operation('click', element=f"{resource_id}{index_desc}", ref=resource_id, index=index)
|
|
1165
|
+
return {"success": True, "message": f"✅ 点击成功: {resource_id}{index_desc}" + (f" (共 {count} 个)" if count > 1 else "")}
|
|
1166
|
+
else:
|
|
1167
|
+
return {"success": False, "message": f"❌ 索引超出范围: 找到 {count} 个元素,但请求索引 {index}"}
|
|
1168
|
+
return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
|
|
1169
|
+
except Exception as e:
|
|
1170
|
+
return {"success": False, "message": f"❌ 点击失败: {e}"}
|
|
1171
|
+
|
|
1172
|
+
# ==================== 长按操作 ====================
|
|
1173
|
+
|
|
1174
|
+
def long_press_at_coords(self, x: int, y: int, duration: float = 1.0,
|
|
1175
|
+
image_width: int = 0, image_height: int = 0,
|
|
1176
|
+
crop_offset_x: int = 0, crop_offset_y: int = 0,
|
|
1177
|
+
original_img_width: int = 0, original_img_height: int = 0) -> Dict:
|
|
1178
|
+
"""长按坐标(核心功能,支持自动坐标转换)
|
|
1179
|
+
|
|
1180
|
+
Args:
|
|
1181
|
+
x: X 坐标(来自截图分析或屏幕坐标)
|
|
1182
|
+
y: Y 坐标(来自截图分析或屏幕坐标)
|
|
1183
|
+
duration: 长按持续时间(秒),默认 1.0
|
|
1184
|
+
image_width: 压缩后图片宽度(AI 看到的图片尺寸)
|
|
1185
|
+
image_height: 压缩后图片高度(AI 看到的图片尺寸)
|
|
1186
|
+
crop_offset_x: 局部截图的 X 偏移量(局部截图时传入)
|
|
1187
|
+
crop_offset_y: 局部截图的 Y 偏移量(局部截图时传入)
|
|
1188
|
+
original_img_width: 截图原始宽度(压缩前的尺寸,用于精确转换)
|
|
1189
|
+
original_img_height: 截图原始高度(压缩前的尺寸,用于精确转换)
|
|
1190
|
+
|
|
1191
|
+
坐标转换说明:
|
|
1192
|
+
1. 全屏压缩截图:AI 坐标 → 原图坐标(基于 image/original_img 比例)
|
|
1193
|
+
2. 局部裁剪截图:AI 坐标 + 偏移量 = 屏幕坐标
|
|
1194
|
+
"""
|
|
1195
|
+
try:
|
|
1196
|
+
# 获取屏幕尺寸
|
|
1197
|
+
screen_width, screen_height = 0, 0
|
|
1198
|
+
if self._is_ios():
|
|
1199
|
+
ios_client = self._get_ios_client()
|
|
1200
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1201
|
+
size = ios_client.wda.window_size()
|
|
1202
|
+
screen_width, screen_height = size[0], size[1]
|
|
1203
|
+
else:
|
|
1204
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
1205
|
+
else:
|
|
1206
|
+
info = self.client.u2.info
|
|
1207
|
+
screen_width = info.get('displayWidth', 0)
|
|
1208
|
+
screen_height = info.get('displayHeight', 0)
|
|
1209
|
+
|
|
1210
|
+
# 🎯 坐标转换
|
|
1211
|
+
original_x, original_y = x, y
|
|
1212
|
+
converted = False
|
|
1213
|
+
conversion_type = ""
|
|
1214
|
+
|
|
1215
|
+
# 情况1:局部裁剪截图 - 加上偏移量
|
|
1216
|
+
if crop_offset_x > 0 or crop_offset_y > 0:
|
|
1217
|
+
x = x + crop_offset_x
|
|
1218
|
+
y = y + crop_offset_y
|
|
1219
|
+
converted = True
|
|
1220
|
+
conversion_type = "crop_offset"
|
|
1221
|
+
# 情况2:全屏压缩截图 - 按比例转换到原图尺寸
|
|
1222
|
+
elif image_width > 0 and image_height > 0:
|
|
1223
|
+
target_width = original_img_width if original_img_width > 0 else screen_width
|
|
1224
|
+
target_height = original_img_height if original_img_height > 0 else screen_height
|
|
1225
|
+
|
|
1226
|
+
if target_width > 0 and target_height > 0:
|
|
1227
|
+
if image_width != target_width or image_height != target_height:
|
|
1228
|
+
x = int(x * target_width / image_width)
|
|
1229
|
+
y = int(y * target_height / image_height)
|
|
1230
|
+
converted = True
|
|
1231
|
+
conversion_type = "scale"
|
|
1232
|
+
|
|
1233
|
+
# 执行长按
|
|
1234
|
+
if self._is_ios():
|
|
1235
|
+
ios_client = self._get_ios_client()
|
|
1236
|
+
# iOS 使用 tap_hold 或 swipe 原地实现长按
|
|
1237
|
+
if hasattr(ios_client.wda, 'tap_hold'):
|
|
1238
|
+
ios_client.wda.tap_hold(x, y, duration=duration)
|
|
1239
|
+
else:
|
|
1240
|
+
# 兜底:用原地 swipe 模拟长按
|
|
1241
|
+
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1242
|
+
else:
|
|
1243
|
+
self.client.u2.long_click(x, y, duration=duration)
|
|
1244
|
+
|
|
1245
|
+
time.sleep(0.3)
|
|
1246
|
+
|
|
1247
|
+
# 计算百分比坐标(用于跨设备兼容)
|
|
1248
|
+
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1249
|
+
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1250
|
+
|
|
1251
|
+
# 记录操作
|
|
1252
|
+
self._record_operation(
|
|
1253
|
+
'long_press',
|
|
1254
|
+
x=x,
|
|
1255
|
+
y=y,
|
|
1256
|
+
x_percent=x_percent,
|
|
1257
|
+
y_percent=y_percent,
|
|
1258
|
+
duration=duration,
|
|
1259
|
+
screen_width=screen_width,
|
|
1260
|
+
screen_height=screen_height,
|
|
1261
|
+
ref=f"coords_{x}_{y}"
|
|
1262
|
+
)
|
|
1263
|
+
|
|
1264
|
+
if converted:
|
|
1265
|
+
if conversion_type == "crop_offset":
|
|
1266
|
+
return {
|
|
1267
|
+
"success": True,
|
|
1268
|
+
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1269
|
+
f" 🔍 局部截图坐标转换: ({original_x},{original_y}) + 偏移({crop_offset_x},{crop_offset_y}) → ({x},{y})"
|
|
1270
|
+
}
|
|
1271
|
+
else:
|
|
1272
|
+
return {
|
|
1273
|
+
"success": True,
|
|
1274
|
+
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s\n"
|
|
1275
|
+
f" 📐 坐标已转换: ({original_x},{original_y}) → ({x},{y})\n"
|
|
1276
|
+
f" 🖼️ 图片尺寸: {image_width}x{image_height} → 屏幕: {screen_width}x{screen_height}"
|
|
1277
|
+
}
|
|
1278
|
+
else:
|
|
1279
|
+
return {
|
|
1280
|
+
"success": True,
|
|
1281
|
+
"message": f"✅ 长按成功: ({x}, {y}) 持续 {duration}s [相对位置: {x_percent}%, {y_percent}%]"
|
|
1282
|
+
}
|
|
1283
|
+
except Exception as e:
|
|
1284
|
+
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1285
|
+
|
|
1286
|
+
def long_press_by_percent(self, x_percent: float, y_percent: float, duration: float = 1.0) -> Dict:
|
|
1287
|
+
"""通过百分比坐标长按(跨设备兼容)
|
|
1288
|
+
|
|
1289
|
+
百分比坐标原理:
|
|
1290
|
+
- 屏幕左上角是 (0%, 0%),右下角是 (100%, 100%)
|
|
1291
|
+
- 屏幕正中央是 (50%, 50%)
|
|
1292
|
+
- 像素坐标 = 屏幕尺寸 × (百分比 / 100)
|
|
1293
|
+
|
|
1294
|
+
Args:
|
|
1295
|
+
x_percent: X轴百分比 (0-100),0=最左,50=中间,100=最右
|
|
1296
|
+
y_percent: Y轴百分比 (0-100),0=最上,50=中间,100=最下
|
|
1297
|
+
duration: 长按持续时间(秒),默认 1.0
|
|
1298
|
+
|
|
1299
|
+
优势:
|
|
1300
|
+
- 同样的百分比在不同分辨率设备上都能点到相同相对位置
|
|
1301
|
+
- 录制一次,多设备回放
|
|
1302
|
+
"""
|
|
1303
|
+
try:
|
|
1304
|
+
# 第1步:获取屏幕尺寸
|
|
1305
|
+
if self._is_ios():
|
|
1306
|
+
ios_client = self._get_ios_client()
|
|
1307
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1308
|
+
size = ios_client.wda.window_size()
|
|
1309
|
+
width, height = size[0], size[1]
|
|
1310
|
+
else:
|
|
1311
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
1312
|
+
else:
|
|
1313
|
+
info = self.client.u2.info
|
|
1314
|
+
width = info.get('displayWidth', 0)
|
|
1315
|
+
height = info.get('displayHeight', 0)
|
|
1316
|
+
|
|
1317
|
+
if width == 0 or height == 0:
|
|
1318
|
+
return {"success": False, "message": "❌ 无法获取屏幕尺寸"}
|
|
1319
|
+
|
|
1320
|
+
# 第2步:百分比转像素坐标
|
|
1321
|
+
x = int(width * x_percent / 100)
|
|
1322
|
+
y = int(height * y_percent / 100)
|
|
1323
|
+
|
|
1324
|
+
# 第3步:执行长按
|
|
1325
|
+
if self._is_ios():
|
|
1326
|
+
ios_client = self._get_ios_client()
|
|
1327
|
+
if hasattr(ios_client.wda, 'tap_hold'):
|
|
1328
|
+
ios_client.wda.tap_hold(x, y, duration=duration)
|
|
1329
|
+
else:
|
|
1330
|
+
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1331
|
+
else:
|
|
1332
|
+
self.client.u2.long_click(x, y, duration=duration)
|
|
1333
|
+
|
|
1334
|
+
time.sleep(0.3)
|
|
1335
|
+
|
|
1336
|
+
# 第4步:记录操作
|
|
1337
|
+
self._record_operation(
|
|
1338
|
+
'long_press',
|
|
1339
|
+
x=x,
|
|
1340
|
+
y=y,
|
|
1341
|
+
x_percent=x_percent,
|
|
1342
|
+
y_percent=y_percent,
|
|
1343
|
+
duration=duration,
|
|
1344
|
+
screen_width=width,
|
|
1345
|
+
screen_height=height,
|
|
1346
|
+
ref=f"percent_{x_percent}_{y_percent}"
|
|
1347
|
+
)
|
|
1348
|
+
|
|
1349
|
+
return {
|
|
1350
|
+
"success": True,
|
|
1351
|
+
"message": f"✅ 百分比长按成功: ({x_percent}%, {y_percent}%) → 像素({x}, {y}) 持续 {duration}s",
|
|
1352
|
+
"screen_size": {"width": width, "height": height},
|
|
1353
|
+
"percent": {"x": x_percent, "y": y_percent},
|
|
1354
|
+
"pixel": {"x": x, "y": y},
|
|
1355
|
+
"duration": duration
|
|
1356
|
+
}
|
|
1357
|
+
except Exception as e:
|
|
1358
|
+
return {"success": False, "message": f"❌ 百分比长按失败: {e}"}
|
|
1359
|
+
|
|
1360
|
+
def long_press_by_text(self, text: str, duration: float = 1.0) -> Dict:
|
|
1361
|
+
"""通过文本长按
|
|
1362
|
+
|
|
1363
|
+
Args:
|
|
1364
|
+
text: 元素的文本内容(精确匹配)
|
|
1365
|
+
duration: 长按持续时间(秒),默认 1.0
|
|
1366
|
+
"""
|
|
1367
|
+
try:
|
|
1368
|
+
if self._is_ios():
|
|
1369
|
+
ios_client = self._get_ios_client()
|
|
1370
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1371
|
+
elem = ios_client.wda(name=text)
|
|
1372
|
+
if not elem.exists:
|
|
1373
|
+
elem = ios_client.wda(label=text)
|
|
1374
|
+
if elem.exists:
|
|
1375
|
+
# iOS 元素长按
|
|
1376
|
+
bounds = elem.bounds
|
|
1377
|
+
x = int((bounds.x + bounds.x + bounds.width) / 2)
|
|
1378
|
+
y = int((bounds.y + bounds.y + bounds.height) / 2)
|
|
1379
|
+
if hasattr(ios_client.wda, 'tap_hold'):
|
|
1380
|
+
ios_client.wda.tap_hold(x, y, duration=duration)
|
|
1381
|
+
else:
|
|
1382
|
+
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1383
|
+
time.sleep(0.3)
|
|
1384
|
+
self._record_operation('long_press', element=text, duration=duration, ref=text)
|
|
1385
|
+
return {"success": True, "message": f"✅ 长按成功: '{text}' 持续 {duration}s"}
|
|
1386
|
+
return {"success": False, "message": f"❌ 文本不存在: {text}"}
|
|
1387
|
+
else:
|
|
1388
|
+
# 先查 XML 树,找到元素
|
|
1389
|
+
found_elem = self._find_element_in_tree(text)
|
|
1390
|
+
|
|
1391
|
+
if found_elem:
|
|
1392
|
+
attr_type = found_elem['attr_type']
|
|
1393
|
+
attr_value = found_elem['attr_value']
|
|
1394
|
+
bounds = found_elem.get('bounds')
|
|
1395
|
+
|
|
1396
|
+
# 根据找到的属性类型,使用对应的选择器
|
|
1397
|
+
if attr_type == 'text':
|
|
1398
|
+
elem = self.client.u2(text=attr_value)
|
|
1399
|
+
elif attr_type == 'textContains':
|
|
1400
|
+
elem = self.client.u2(textContains=attr_value)
|
|
1401
|
+
elif attr_type == 'description':
|
|
1402
|
+
elem = self.client.u2(description=attr_value)
|
|
1403
|
+
elif attr_type == 'descriptionContains':
|
|
1404
|
+
elem = self.client.u2(descriptionContains=attr_value)
|
|
1405
|
+
else:
|
|
1406
|
+
elem = None
|
|
1407
|
+
|
|
1408
|
+
if elem and elem.exists(timeout=1):
|
|
1409
|
+
elem.long_click(duration=duration)
|
|
1410
|
+
time.sleep(0.3)
|
|
1411
|
+
self._record_operation('long_press', element=text, duration=duration, ref=f"{attr_type}:{attr_value}")
|
|
1412
|
+
return {"success": True, "message": f"✅ 长按成功({attr_type}): '{text}' 持续 {duration}s"}
|
|
1413
|
+
|
|
1414
|
+
# 如果选择器失败,用坐标兜底
|
|
1415
|
+
if bounds:
|
|
1416
|
+
x = (bounds[0] + bounds[2]) // 2
|
|
1417
|
+
y = (bounds[1] + bounds[3]) // 2
|
|
1418
|
+
self.client.u2.long_click(x, y, duration=duration)
|
|
1419
|
+
time.sleep(0.3)
|
|
1420
|
+
self._record_operation('long_press', element=text, x=x, y=y, duration=duration, ref=f"coords:{x},{y}")
|
|
1421
|
+
return {"success": True, "message": f"✅ 长按成功(坐标兜底): '{text}' @ ({x},{y}) 持续 {duration}s"}
|
|
1422
|
+
|
|
1423
|
+
return {"success": False, "message": f"❌ 文本不存在: {text}"}
|
|
1424
|
+
except Exception as e:
|
|
1425
|
+
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1426
|
+
|
|
1427
|
+
def long_press_by_id(self, resource_id: str, duration: float = 1.0) -> Dict:
|
|
1428
|
+
"""通过 resource-id 长按
|
|
1429
|
+
|
|
1430
|
+
Args:
|
|
1431
|
+
resource_id: 元素的 resource-id
|
|
1432
|
+
duration: 长按持续时间(秒),默认 1.0
|
|
1433
|
+
"""
|
|
1434
|
+
try:
|
|
1435
|
+
if self._is_ios():
|
|
1436
|
+
ios_client = self._get_ios_client()
|
|
1437
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1438
|
+
elem = ios_client.wda(id=resource_id)
|
|
1439
|
+
if not elem.exists:
|
|
1440
|
+
elem = ios_client.wda(name=resource_id)
|
|
1441
|
+
if elem.exists:
|
|
1442
|
+
bounds = elem.bounds
|
|
1443
|
+
x = int((bounds.x + bounds.x + bounds.width) / 2)
|
|
1444
|
+
y = int((bounds.y + bounds.y + bounds.height) / 2)
|
|
1445
|
+
if hasattr(ios_client.wda, 'tap_hold'):
|
|
1446
|
+
ios_client.wda.tap_hold(x, y, duration=duration)
|
|
1447
|
+
else:
|
|
1448
|
+
ios_client.wda.swipe(x, y, x, y, duration=duration)
|
|
1449
|
+
time.sleep(0.3)
|
|
1450
|
+
self._record_operation('long_press', element=resource_id, duration=duration, ref=resource_id)
|
|
1451
|
+
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
1452
|
+
return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
|
|
1453
|
+
else:
|
|
1454
|
+
elem = self.client.u2(resourceId=resource_id)
|
|
1455
|
+
if elem.exists(timeout=0.5):
|
|
1456
|
+
elem.long_click(duration=duration)
|
|
1457
|
+
time.sleep(0.3)
|
|
1458
|
+
self._record_operation('long_press', element=resource_id, duration=duration, ref=resource_id)
|
|
1459
|
+
return {"success": True, "message": f"✅ 长按成功: {resource_id} 持续 {duration}s"}
|
|
1460
|
+
return {"success": False, "message": f"❌ 元素不存在: {resource_id}"}
|
|
1461
|
+
except Exception as e:
|
|
1462
|
+
return {"success": False, "message": f"❌ 长按失败: {e}"}
|
|
1463
|
+
|
|
1464
|
+
# ==================== 输入操作 ====================
|
|
1465
|
+
|
|
1466
|
+
def input_text_by_id(self, resource_id: str, text: str) -> Dict:
|
|
1467
|
+
"""通过 resource-id 输入文本
|
|
1468
|
+
|
|
1469
|
+
优化策略:
|
|
1470
|
+
1. 先用 resourceId 定位
|
|
1471
|
+
2. 如果只有 1 个元素 → 直接输入
|
|
1472
|
+
3. 如果有多个相同 ID(>5个说明 ID 不可靠)→ 改用 EditText 类型定位
|
|
1473
|
+
4. 多个 EditText 时选择最靠上的(搜索框通常在顶部)
|
|
1474
|
+
"""
|
|
1475
|
+
try:
|
|
1476
|
+
if self._is_ios():
|
|
1477
|
+
ios_client = self._get_ios_client()
|
|
1478
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1479
|
+
elem = ios_client.wda(id=resource_id)
|
|
1480
|
+
if not elem.exists:
|
|
1481
|
+
elem = ios_client.wda(name=resource_id)
|
|
1482
|
+
if elem.exists:
|
|
1483
|
+
elem.set_text(text)
|
|
1484
|
+
time.sleep(0.3)
|
|
1485
|
+
self._record_operation('input', element=resource_id, ref=resource_id, text=text)
|
|
1486
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}'"}
|
|
1487
|
+
return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
|
|
1488
|
+
else:
|
|
1489
|
+
elements = self.client.u2(resourceId=resource_id)
|
|
1490
|
+
|
|
1491
|
+
# 检查是否存在
|
|
1492
|
+
if elements.exists(timeout=0.5):
|
|
1493
|
+
count = elements.count
|
|
1494
|
+
|
|
1495
|
+
# 只有 1 个元素,直接输入
|
|
1496
|
+
if count == 1:
|
|
1497
|
+
elements.set_text(text)
|
|
1498
|
+
time.sleep(0.3)
|
|
1499
|
+
self._record_operation('input', element=resource_id, ref=resource_id, text=text)
|
|
1500
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}'"}
|
|
1501
|
+
|
|
1502
|
+
# 多个相同 ID(<=5个),尝试智能选择
|
|
1503
|
+
if count <= 5:
|
|
1504
|
+
for i in range(count):
|
|
1505
|
+
try:
|
|
1506
|
+
elem = elements[i]
|
|
1507
|
+
info = elem.info
|
|
1508
|
+
# 优先选择可编辑的
|
|
1509
|
+
if info.get('editable') or info.get('focusable'):
|
|
1510
|
+
elem.set_text(text)
|
|
1511
|
+
time.sleep(0.3)
|
|
1512
|
+
self._record_operation('input', element=resource_id, ref=resource_id, text=text)
|
|
1513
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}'"}
|
|
1514
|
+
except:
|
|
1515
|
+
continue
|
|
1516
|
+
# 没找到可编辑的,用第一个
|
|
1517
|
+
elements[0].set_text(text)
|
|
1518
|
+
time.sleep(0.3)
|
|
1519
|
+
self._record_operation('input', element=resource_id, ref=resource_id, text=text)
|
|
1520
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}'"}
|
|
1521
|
+
|
|
1522
|
+
# ID 不可靠(不存在或太多),改用 EditText 类型定位
|
|
1523
|
+
edit_texts = self.client.u2(className='android.widget.EditText')
|
|
1524
|
+
if edit_texts.exists(timeout=0.5):
|
|
1525
|
+
et_count = edit_texts.count
|
|
1526
|
+
if et_count == 1:
|
|
1527
|
+
edit_texts.set_text(text)
|
|
1528
|
+
time.sleep(0.3)
|
|
1529
|
+
self._record_operation('input', element='EditText', ref='EditText', text=text)
|
|
1530
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}' (通过 EditText 定位)"}
|
|
1531
|
+
|
|
1532
|
+
# 多个 EditText,选择最靠上的
|
|
1533
|
+
best_elem = None
|
|
1534
|
+
min_top = 9999
|
|
1535
|
+
for i in range(et_count):
|
|
1536
|
+
try:
|
|
1537
|
+
elem = edit_texts[i]
|
|
1538
|
+
top = elem.info.get('bounds', {}).get('top', 9999)
|
|
1539
|
+
if top < min_top:
|
|
1540
|
+
min_top = top
|
|
1541
|
+
best_elem = elem
|
|
1542
|
+
except:
|
|
1543
|
+
continue
|
|
1544
|
+
|
|
1545
|
+
if best_elem:
|
|
1546
|
+
best_elem.set_text(text)
|
|
1547
|
+
time.sleep(0.3)
|
|
1548
|
+
self._record_operation('input', element='EditText', ref='EditText', text=text)
|
|
1549
|
+
return {"success": True, "message": f"✅ 输入成功: '{text}' (通过 EditText 定位,选择最顶部的)"}
|
|
1550
|
+
|
|
1551
|
+
return {"success": False, "message": f"❌ 输入框不存在: {resource_id}"}
|
|
1552
|
+
|
|
1553
|
+
except Exception as e:
|
|
1554
|
+
return {"success": False, "message": f"❌ 输入失败: {e}"}
|
|
1555
|
+
|
|
1556
|
+
def input_at_coords(self, x: int, y: int, text: str) -> Dict:
|
|
1557
|
+
"""点击坐标后输入文本(适合游戏)"""
|
|
1558
|
+
try:
|
|
1559
|
+
# 获取屏幕尺寸(用于转换百分比)
|
|
1560
|
+
screen_width, screen_height = 0, 0
|
|
1561
|
+
|
|
1562
|
+
# 先点击聚焦
|
|
1563
|
+
if self._is_ios():
|
|
1564
|
+
ios_client = self._get_ios_client()
|
|
1565
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1566
|
+
ios_client.wda.click(x, y)
|
|
1567
|
+
size = ios_client.wda.window_size()
|
|
1568
|
+
screen_width, screen_height = size[0], size[1]
|
|
1569
|
+
else:
|
|
1570
|
+
self.client.u2.click(x, y)
|
|
1571
|
+
info = self.client.u2.info
|
|
1572
|
+
screen_width = info.get('displayWidth', 0)
|
|
1573
|
+
screen_height = info.get('displayHeight', 0)
|
|
1574
|
+
|
|
1575
|
+
time.sleep(0.3)
|
|
1576
|
+
|
|
1577
|
+
# 输入文本
|
|
1578
|
+
if self._is_ios():
|
|
1579
|
+
ios_client = self._get_ios_client()
|
|
1580
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1581
|
+
ios_client.wda.send_keys(text)
|
|
1582
|
+
else:
|
|
1583
|
+
self.client.u2.send_keys(text)
|
|
1584
|
+
|
|
1585
|
+
time.sleep(0.3)
|
|
1586
|
+
|
|
1587
|
+
# 计算百分比坐标
|
|
1588
|
+
x_percent = round(x / screen_width * 100, 1) if screen_width > 0 else 0
|
|
1589
|
+
y_percent = round(y / screen_height * 100, 1) if screen_height > 0 else 0
|
|
1590
|
+
|
|
1591
|
+
self._record_operation(
|
|
1592
|
+
'input',
|
|
1593
|
+
x=x,
|
|
1594
|
+
y=y,
|
|
1595
|
+
x_percent=x_percent,
|
|
1596
|
+
y_percent=y_percent,
|
|
1597
|
+
ref=f"coords_{x}_{y}",
|
|
1598
|
+
text=text
|
|
1599
|
+
)
|
|
1600
|
+
|
|
1601
|
+
return {"success": True, "message": f"✅ 输入成功: ({x}, {y}) [相对位置: {x_percent}%, {y_percent}%] -> '{text}'"}
|
|
1602
|
+
except Exception as e:
|
|
1603
|
+
return {"success": False, "message": f"❌ 输入失败: {e}"}
|
|
1604
|
+
|
|
1605
|
+
# ==================== 导航操作 ====================
|
|
1606
|
+
|
|
1607
|
+
async def swipe(self, direction: str, y: Optional[int] = None, y_percent: Optional[float] = None) -> Dict:
|
|
1608
|
+
"""滑动屏幕
|
|
1609
|
+
|
|
1610
|
+
Args:
|
|
1611
|
+
direction: 滑动方向 (up/down/left/right)
|
|
1612
|
+
y: 左右滑动时指定的高度坐标(像素)
|
|
1613
|
+
y_percent: 左右滑动时指定的高度百分比 (0-100)
|
|
1614
|
+
"""
|
|
1615
|
+
try:
|
|
1616
|
+
if self._is_ios():
|
|
1617
|
+
ios_client = self._get_ios_client()
|
|
1618
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1619
|
+
size = ios_client.wda.window_size()
|
|
1620
|
+
width, height = size[0], size[1]
|
|
1621
|
+
else:
|
|
1622
|
+
return {"success": False, "message": "❌ iOS 客户端未初始化"}
|
|
1623
|
+
else:
|
|
1624
|
+
width, height = self.client.u2.window_size()
|
|
1625
|
+
|
|
1626
|
+
center_x, center_y = width // 2, height // 2
|
|
1627
|
+
|
|
1628
|
+
# 对于左右滑动,如果指定了 y 或 y_percent,使用指定的高度
|
|
1629
|
+
if direction in ['left', 'right']:
|
|
1630
|
+
if y_percent is not None:
|
|
1631
|
+
if not (0 <= y_percent <= 100):
|
|
1632
|
+
return {"success": False, "message": f"❌ y_percent 必须在 0-100 之间: {y_percent}"}
|
|
1633
|
+
swipe_y = int(height * y_percent / 100)
|
|
1634
|
+
elif y is not None:
|
|
1635
|
+
if not (0 <= y <= height):
|
|
1636
|
+
return {"success": False, "message": f"❌ y 坐标超出屏幕范围 (0-{height}): {y}"}
|
|
1637
|
+
swipe_y = y
|
|
1638
|
+
else:
|
|
1639
|
+
swipe_y = center_y
|
|
1640
|
+
else:
|
|
1641
|
+
swipe_y = center_y
|
|
1642
|
+
|
|
1643
|
+
swipe_map = {
|
|
1644
|
+
'up': (center_x, int(height * 0.8), center_x, int(height * 0.2)),
|
|
1645
|
+
'down': (center_x, int(height * 0.2), center_x, int(height * 0.8)),
|
|
1646
|
+
'left': (int(width * 0.8), swipe_y, int(width * 0.2), swipe_y),
|
|
1647
|
+
'right': (int(width * 0.2), swipe_y, int(width * 0.8), swipe_y),
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
if direction not in swipe_map:
|
|
1651
|
+
return {"success": False, "message": f"❌ 不支持的方向: {direction}"}
|
|
1652
|
+
|
|
1653
|
+
x1, y1, x2, y2 = swipe_map[direction]
|
|
1654
|
+
|
|
1655
|
+
if self._is_ios():
|
|
1656
|
+
ios_client.wda.swipe(x1, y1, x2, y2)
|
|
1657
|
+
else:
|
|
1658
|
+
self.client.u2.swipe(x1, y1, x2, y2, duration=0.5)
|
|
1659
|
+
|
|
1660
|
+
# 记录操作信息
|
|
1661
|
+
record_info = {'direction': direction}
|
|
1662
|
+
if y is not None:
|
|
1663
|
+
record_info['y'] = y
|
|
1664
|
+
if y_percent is not None:
|
|
1665
|
+
record_info['y_percent'] = y_percent
|
|
1666
|
+
self._record_operation('swipe', **record_info)
|
|
1667
|
+
|
|
1668
|
+
# 构建返回消息
|
|
1669
|
+
msg = f"✅ 滑动成功: {direction}"
|
|
1670
|
+
if direction in ['left', 'right']:
|
|
1671
|
+
if y_percent is not None:
|
|
1672
|
+
msg += f" (高度: {y_percent}% = {swipe_y}px)"
|
|
1673
|
+
elif y is not None:
|
|
1674
|
+
msg += f" (高度: {y}px)"
|
|
1675
|
+
|
|
1676
|
+
return {"success": True, "message": msg}
|
|
1677
|
+
except Exception as e:
|
|
1678
|
+
return {"success": False, "message": f"❌ 滑动失败: {e}"}
|
|
1679
|
+
|
|
1680
|
+
async def press_key(self, key: str) -> Dict:
|
|
1681
|
+
"""按键操作"""
|
|
1682
|
+
key_map = {
|
|
1683
|
+
'enter': 66, '回车': 66,
|
|
1684
|
+
'search': 84, '搜索': 84,
|
|
1685
|
+
'back': 4, '返回': 4,
|
|
1686
|
+
'home': 3,
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
try:
|
|
1690
|
+
if self._is_ios():
|
|
1691
|
+
ios_key_map = {'enter': 'return', 'back': 'back', 'home': 'home'}
|
|
1692
|
+
ios_key = ios_key_map.get(key.lower())
|
|
1693
|
+
if ios_key:
|
|
1694
|
+
ios_client = self._get_ios_client()
|
|
1695
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1696
|
+
# iOS 使用不同的按键方式
|
|
1697
|
+
if ios_key == 'return':
|
|
1698
|
+
ios_client.wda.send_keys('\n')
|
|
1699
|
+
elif ios_key == 'home':
|
|
1700
|
+
ios_client.wda.home()
|
|
1701
|
+
return {"success": True, "message": f"✅ 按键成功: {key}"}
|
|
1702
|
+
return {"success": False, "message": f"❌ iOS 不支持: {key}"}
|
|
1703
|
+
else:
|
|
1704
|
+
keycode = key_map.get(key.lower())
|
|
1705
|
+
if keycode:
|
|
1706
|
+
self.client.u2.shell(f'input keyevent {keycode}')
|
|
1707
|
+
self._record_operation('press_key', key=key)
|
|
1708
|
+
return {"success": True, "message": f"✅ 按键成功: {key}"}
|
|
1709
|
+
return {"success": False, "message": f"❌ 不支持的按键: {key}"}
|
|
1710
|
+
except Exception as e:
|
|
1711
|
+
return {"success": False, "message": f"❌ 按键失败: {e}"}
|
|
1712
|
+
|
|
1713
|
+
def wait(self, seconds: float) -> Dict:
|
|
1714
|
+
"""等待指定时间"""
|
|
1715
|
+
time.sleep(seconds)
|
|
1716
|
+
return {"success": True, "message": f"✅ 已等待 {seconds} 秒"}
|
|
1717
|
+
|
|
1718
|
+
# ==================== 应用管理 ====================
|
|
1719
|
+
|
|
1720
|
+
async def launch_app(self, package_name: str) -> Dict:
|
|
1721
|
+
"""启动应用"""
|
|
1722
|
+
try:
|
|
1723
|
+
if self._is_ios():
|
|
1724
|
+
ios_client = self._get_ios_client()
|
|
1725
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1726
|
+
ios_client.wda.app_activate(package_name)
|
|
1727
|
+
else:
|
|
1728
|
+
self.client.u2.app_start(package_name)
|
|
1729
|
+
|
|
1730
|
+
await asyncio.sleep(2)
|
|
1731
|
+
|
|
1732
|
+
self._record_operation('launch_app', package_name=package_name)
|
|
1733
|
+
|
|
1734
|
+
return {
|
|
1735
|
+
"success": True,
|
|
1736
|
+
"message": f"✅ 已启动: {package_name}\n💡 建议等待 2-3 秒让页面加载"
|
|
1737
|
+
}
|
|
1738
|
+
except Exception as e:
|
|
1739
|
+
return {"success": False, "message": f"❌ 启动失败: {e}"}
|
|
1740
|
+
|
|
1741
|
+
def terminate_app(self, package_name: str) -> Dict:
|
|
1742
|
+
"""终止应用"""
|
|
1743
|
+
try:
|
|
1744
|
+
if self._is_ios():
|
|
1745
|
+
ios_client = self._get_ios_client()
|
|
1746
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1747
|
+
ios_client.wda.app_terminate(package_name)
|
|
1748
|
+
else:
|
|
1749
|
+
self.client.u2.app_stop(package_name)
|
|
1750
|
+
return {"success": True, "message": f"✅ 已终止: {package_name}"}
|
|
1751
|
+
except Exception as e:
|
|
1752
|
+
return {"success": False, "message": f"❌ 终止失败: {e}"}
|
|
1753
|
+
|
|
1754
|
+
def list_apps(self, filter_keyword: str = "") -> Dict:
|
|
1755
|
+
"""列出已安装应用"""
|
|
1756
|
+
try:
|
|
1757
|
+
if self._is_ios():
|
|
1758
|
+
ios_client = self._get_ios_client()
|
|
1759
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1760
|
+
# iOS 暂不支持列出所有应用
|
|
1761
|
+
return {
|
|
1762
|
+
"success": True,
|
|
1763
|
+
"apps": [],
|
|
1764
|
+
"count": 0,
|
|
1765
|
+
"message": "💡 iOS 暂不支持列出所有应用,请直接使用 bundle_id 启动"
|
|
1766
|
+
}
|
|
1767
|
+
else:
|
|
1768
|
+
apps = self.client.u2.app_list()
|
|
1769
|
+
if filter_keyword:
|
|
1770
|
+
apps = [app for app in apps if filter_keyword.lower() in app.lower()]
|
|
1771
|
+
return {
|
|
1772
|
+
"success": True,
|
|
1773
|
+
"apps": apps[:50], # 限制返回数量
|
|
1774
|
+
"count": len(apps)
|
|
1775
|
+
}
|
|
1776
|
+
except Exception as e:
|
|
1777
|
+
return {"success": False, "message": f"❌ 获取应用列表失败: {e}"}
|
|
1778
|
+
|
|
1779
|
+
# ==================== 设备管理 ====================
|
|
1780
|
+
|
|
1781
|
+
def list_devices(self) -> Dict:
|
|
1782
|
+
"""列出已连接设备"""
|
|
1783
|
+
try:
|
|
1784
|
+
platform = "ios" if self._is_ios() else "android"
|
|
1785
|
+
|
|
1786
|
+
if platform == "ios":
|
|
1787
|
+
from .ios_device_manager_wda import IOSDeviceManagerWDA
|
|
1788
|
+
manager = IOSDeviceManagerWDA()
|
|
1789
|
+
devices = manager.list_devices()
|
|
1790
|
+
else:
|
|
1791
|
+
from .device_manager import DeviceManager
|
|
1792
|
+
manager = DeviceManager()
|
|
1793
|
+
devices = manager.list_devices()
|
|
1794
|
+
|
|
1795
|
+
return {
|
|
1796
|
+
"success": True,
|
|
1797
|
+
"platform": platform,
|
|
1798
|
+
"devices": devices,
|
|
1799
|
+
"count": len(devices)
|
|
1800
|
+
}
|
|
1801
|
+
except Exception as e:
|
|
1802
|
+
return {"success": False, "message": f"❌ 获取设备列表失败: {e}"}
|
|
1803
|
+
|
|
1804
|
+
def check_connection(self) -> Dict:
|
|
1805
|
+
"""检查设备连接"""
|
|
1806
|
+
try:
|
|
1807
|
+
if self._is_ios():
|
|
1808
|
+
ios_client = self._get_ios_client()
|
|
1809
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
1810
|
+
return {"success": True, "connected": True, "platform": "ios"}
|
|
1811
|
+
return {"success": False, "connected": False, "message": "❌ iOS 未连接"}
|
|
1812
|
+
else:
|
|
1813
|
+
info = self.client.u2.device_info
|
|
1814
|
+
return {
|
|
1815
|
+
"success": True,
|
|
1816
|
+
"connected": True,
|
|
1817
|
+
"platform": "android",
|
|
1818
|
+
"device": f"{info.get('brand', '')} {info.get('model', '')}"
|
|
1819
|
+
}
|
|
1820
|
+
except Exception as e:
|
|
1821
|
+
return {"success": False, "connected": False, "message": f"❌ 连接检查失败: {e}"}
|
|
1822
|
+
|
|
1823
|
+
# ==================== 辅助工具 ====================
|
|
1824
|
+
|
|
1825
|
+
def list_elements(self) -> List[Dict]:
|
|
1826
|
+
"""列出页面元素"""
|
|
1827
|
+
try:
|
|
1828
|
+
if self._is_ios():
|
|
1829
|
+
ios_client = self._get_ios_client()
|
|
1830
|
+
if ios_client and hasattr(ios_client, 'list_elements'):
|
|
1831
|
+
return ios_client.list_elements()
|
|
1832
|
+
return [{"error": "iOS 暂不支持元素列表,建议使用截图"}]
|
|
1833
|
+
else:
|
|
1834
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
1835
|
+
elements = self.client.xml_parser.parse(xml_string)
|
|
1836
|
+
|
|
1837
|
+
result = []
|
|
1838
|
+
for elem in elements:
|
|
1839
|
+
if elem.get('clickable') or elem.get('focusable'):
|
|
1840
|
+
result.append({
|
|
1841
|
+
'resource_id': elem.get('resource_id', ''),
|
|
1842
|
+
'text': elem.get('text', ''),
|
|
1843
|
+
'content_desc': elem.get('content_desc', ''),
|
|
1844
|
+
'bounds': elem.get('bounds', ''),
|
|
1845
|
+
'clickable': elem.get('clickable', False)
|
|
1846
|
+
})
|
|
1847
|
+
return result
|
|
1848
|
+
except Exception as e:
|
|
1849
|
+
return [{"error": f"获取元素失败: {e}"}]
|
|
1850
|
+
|
|
1851
|
+
def find_close_button(self) -> Dict:
|
|
1852
|
+
"""智能查找关闭按钮(不点击,只返回位置)
|
|
1853
|
+
|
|
1854
|
+
从元素列表中找最可能的关闭按钮,返回其坐标和百分比位置。
|
|
1855
|
+
适用于关闭弹窗广告等场景。
|
|
1856
|
+
|
|
1857
|
+
Returns:
|
|
1858
|
+
包含关闭按钮位置信息的字典,或截图让 AI 分析
|
|
1859
|
+
"""
|
|
1860
|
+
try:
|
|
1861
|
+
import re
|
|
1862
|
+
|
|
1863
|
+
if self._is_ios():
|
|
1864
|
+
return {"success": False, "message": "iOS 暂不支持,请使用截图+坐标点击"}
|
|
1865
|
+
|
|
1866
|
+
# 获取屏幕尺寸
|
|
1867
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
1868
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
1869
|
+
|
|
1870
|
+
# 获取元素列表
|
|
1871
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
1872
|
+
import xml.etree.ElementTree as ET
|
|
1873
|
+
root = ET.fromstring(xml_string)
|
|
1874
|
+
|
|
1875
|
+
# 关闭按钮特征
|
|
1876
|
+
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', '跳过', '知道了', '我知道了']
|
|
1877
|
+
candidates = []
|
|
1878
|
+
|
|
1879
|
+
for elem in root.iter():
|
|
1880
|
+
text = elem.attrib.get('text', '')
|
|
1881
|
+
content_desc = elem.attrib.get('content-desc', '')
|
|
1882
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
1883
|
+
class_name = elem.attrib.get('class', '')
|
|
1884
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
1885
|
+
|
|
1886
|
+
if not bounds_str:
|
|
1887
|
+
continue
|
|
1888
|
+
|
|
1889
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
1890
|
+
if not match:
|
|
1891
|
+
continue
|
|
1892
|
+
|
|
1893
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
1894
|
+
width = x2 - x1
|
|
1895
|
+
height = y2 - y1
|
|
1896
|
+
center_x = (x1 + x2) // 2
|
|
1897
|
+
center_y = (y1 + y2) // 2
|
|
1898
|
+
|
|
1899
|
+
# 计算百分比
|
|
1900
|
+
x_percent = round(center_x / screen_width * 100, 1)
|
|
1901
|
+
y_percent = round(center_y / screen_height * 100, 1)
|
|
1902
|
+
|
|
1903
|
+
score = 0
|
|
1904
|
+
reason = ""
|
|
1905
|
+
|
|
1906
|
+
# 策略1:关闭文本
|
|
1907
|
+
if text in close_texts:
|
|
1908
|
+
score = 100
|
|
1909
|
+
reason = f"文本='{text}'"
|
|
1910
|
+
|
|
1911
|
+
# 策略2:content-desc 包含关闭关键词
|
|
1912
|
+
elif any(kw in content_desc.lower() for kw in ['关闭', 'close', 'dismiss', '跳过']):
|
|
1913
|
+
score = 90
|
|
1914
|
+
reason = f"描述='{content_desc}'"
|
|
1915
|
+
|
|
1916
|
+
# 策略3:小尺寸的 clickable 元素(可能是 X 图标)
|
|
1917
|
+
elif clickable:
|
|
1918
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
1919
|
+
max_size = max(120, int(screen_width * 0.12))
|
|
1920
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
1921
|
+
# 基于位置评分:角落位置加分
|
|
1922
|
+
rel_x = center_x / screen_width
|
|
1923
|
+
rel_y = center_y / screen_height
|
|
1924
|
+
|
|
1925
|
+
# 右上角得分最高
|
|
1926
|
+
if rel_x > 0.6 and rel_y < 0.5:
|
|
1927
|
+
score = 70 + (rel_x - 0.6) * 50 + (0.5 - rel_y) * 50
|
|
1928
|
+
reason = f"右上角小元素 {width}x{height}px"
|
|
1929
|
+
# 左上角
|
|
1930
|
+
elif rel_x < 0.4 and rel_y < 0.5:
|
|
1931
|
+
score = 60 + (0.4 - rel_x) * 50 + (0.5 - rel_y) * 50
|
|
1932
|
+
reason = f"左上角小元素 {width}x{height}px"
|
|
1933
|
+
# 其他位置的小元素
|
|
1934
|
+
elif 'Image' in class_name:
|
|
1935
|
+
score = 50
|
|
1936
|
+
reason = f"图片元素 {width}x{height}px"
|
|
1937
|
+
else:
|
|
1938
|
+
score = 40
|
|
1939
|
+
reason = f"小型可点击元素 {width}x{height}px"
|
|
1940
|
+
|
|
1941
|
+
if score > 0:
|
|
1942
|
+
candidates.append({
|
|
1943
|
+
'score': score,
|
|
1944
|
+
'reason': reason,
|
|
1945
|
+
'bounds': bounds_str,
|
|
1946
|
+
'center_x': center_x,
|
|
1947
|
+
'center_y': center_y,
|
|
1948
|
+
'x_percent': x_percent,
|
|
1949
|
+
'y_percent': y_percent,
|
|
1950
|
+
'size': f"{width}x{height}"
|
|
1951
|
+
})
|
|
1952
|
+
|
|
1953
|
+
if not candidates:
|
|
1954
|
+
# 没找到,截图让 AI 分析
|
|
1955
|
+
screenshot_result = self.take_screenshot(description="找关闭按钮", compress=True)
|
|
1956
|
+
return {
|
|
1957
|
+
"success": False,
|
|
1958
|
+
"message": "❌ 元素树未找到关闭按钮,已截图供 AI 分析",
|
|
1959
|
+
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
1960
|
+
"screen_size": {"width": screen_width, "height": screen_height},
|
|
1961
|
+
"image_size": {
|
|
1962
|
+
"width": screenshot_result.get("image_width"),
|
|
1963
|
+
"height": screenshot_result.get("image_height")
|
|
1964
|
+
},
|
|
1965
|
+
"original_size": {
|
|
1966
|
+
"width": screenshot_result.get("original_img_width"),
|
|
1967
|
+
"height": screenshot_result.get("original_img_height")
|
|
1968
|
+
},
|
|
1969
|
+
"tip": "请分析截图找到 X 关闭按钮,然后调用 mobile_click_by_percent(x_percent, y_percent)"
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
# 按得分排序
|
|
1973
|
+
candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
1974
|
+
best = candidates[0]
|
|
1975
|
+
|
|
1976
|
+
return {
|
|
1977
|
+
"success": True,
|
|
1978
|
+
"message": f"✅ 找到可能的关闭按钮",
|
|
1979
|
+
"best_candidate": {
|
|
1980
|
+
"reason": best['reason'],
|
|
1981
|
+
"center": {"x": best['center_x'], "y": best['center_y']},
|
|
1982
|
+
"percent": {"x": best['x_percent'], "y": best['y_percent']},
|
|
1983
|
+
"bounds": best['bounds'],
|
|
1984
|
+
"size": best['size'],
|
|
1985
|
+
"score": best['score']
|
|
1986
|
+
},
|
|
1987
|
+
"click_command": f"mobile_click_by_percent({best['x_percent']}, {best['y_percent']})",
|
|
1988
|
+
"other_candidates": [
|
|
1989
|
+
{"reason": c['reason'], "percent": f"({c['x_percent']}%, {c['y_percent']}%)", "score": c['score']}
|
|
1990
|
+
for c in candidates[1:4]
|
|
1991
|
+
] if len(candidates) > 1 else [],
|
|
1992
|
+
"screen_size": {"width": screen_width, "height": screen_height}
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
except Exception as e:
|
|
1996
|
+
return {"success": False, "message": f"❌ 查找关闭按钮失败: {e}"}
|
|
1997
|
+
|
|
1998
|
+
def close_popup(self) -> Dict:
|
|
1999
|
+
"""智能关闭弹窗(改进版)
|
|
2000
|
+
|
|
2001
|
+
核心改进:先检测弹窗区域,再在弹窗范围内查找关闭按钮
|
|
2002
|
+
|
|
2003
|
+
策略(优先级从高到低):
|
|
2004
|
+
1. 检测弹窗区域(非全屏的大面积容器)
|
|
2005
|
+
2. 在弹窗边界内查找关闭相关的文本/描述(×、X、关闭、close 等)
|
|
2006
|
+
3. 在弹窗边界内查找小尺寸的 clickable 元素(优先边角位置)
|
|
2007
|
+
4. 如果都找不到,截图让 AI 视觉识别
|
|
2008
|
+
|
|
2009
|
+
适配策略:
|
|
2010
|
+
- X 按钮可能在任意位置(上下左右都支持)
|
|
2011
|
+
- 使用百分比坐标记录,跨分辨率兼容
|
|
2012
|
+
"""
|
|
2013
|
+
try:
|
|
2014
|
+
import re
|
|
2015
|
+
import xml.etree.ElementTree as ET
|
|
2016
|
+
|
|
2017
|
+
# 获取屏幕尺寸
|
|
2018
|
+
if self._is_ios():
|
|
2019
|
+
return {"success": False, "message": "iOS 暂不支持,请使用截图+坐标点击"}
|
|
2020
|
+
|
|
2021
|
+
screen_width = self.client.u2.info.get('displayWidth', 720)
|
|
2022
|
+
screen_height = self.client.u2.info.get('displayHeight', 1280)
|
|
2023
|
+
|
|
2024
|
+
# 获取原始 XML
|
|
2025
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2026
|
+
|
|
2027
|
+
# 关闭按钮的文本特征
|
|
2028
|
+
close_texts = ['×', 'X', 'x', '关闭', '取消', 'close', 'Close', 'CLOSE', '跳过', '知道了']
|
|
2029
|
+
close_desc_keywords = ['关闭', 'close', 'dismiss', 'cancel', '跳过']
|
|
2030
|
+
|
|
2031
|
+
close_candidates = []
|
|
2032
|
+
popup_bounds = None # 弹窗区域
|
|
2033
|
+
|
|
2034
|
+
# 解析 XML
|
|
2035
|
+
try:
|
|
2036
|
+
root = ET.fromstring(xml_string)
|
|
2037
|
+
all_elements = list(root.iter())
|
|
2038
|
+
|
|
2039
|
+
# ===== 第一步:检测弹窗区域 =====
|
|
2040
|
+
# 弹窗特征:非全屏、面积较大、通常在屏幕中央的容器
|
|
2041
|
+
popup_containers = []
|
|
2042
|
+
for idx, elem in enumerate(all_elements):
|
|
2043
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2044
|
+
class_name = elem.attrib.get('class', '')
|
|
2045
|
+
|
|
2046
|
+
if not bounds_str:
|
|
2047
|
+
continue
|
|
2048
|
+
|
|
2049
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2050
|
+
if not match:
|
|
2051
|
+
continue
|
|
2052
|
+
|
|
2053
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2054
|
+
width = x2 - x1
|
|
2055
|
+
height = y2 - y1
|
|
2056
|
+
area = width * height
|
|
2057
|
+
screen_area = screen_width * screen_height
|
|
2058
|
+
|
|
2059
|
+
# 弹窗容器特征:
|
|
2060
|
+
# 1. 面积在屏幕的 10%-90% 之间(非全屏)
|
|
2061
|
+
# 2. 宽度或高度不等于屏幕尺寸
|
|
2062
|
+
# 3. 是容器类型(Layout/View/Dialog)
|
|
2063
|
+
is_container = any(kw in class_name for kw in ['Layout', 'View', 'Dialog', 'Card', 'Container'])
|
|
2064
|
+
area_ratio = area / screen_area
|
|
2065
|
+
is_not_fullscreen = (width < screen_width * 0.98 or height < screen_height * 0.98)
|
|
2066
|
+
is_reasonable_size = 0.08 < area_ratio < 0.9
|
|
2067
|
+
|
|
2068
|
+
# 排除状态栏区域(y1 通常很小)
|
|
2069
|
+
is_below_statusbar = y1 > 50
|
|
2070
|
+
|
|
2071
|
+
if is_container and is_not_fullscreen and is_reasonable_size and is_below_statusbar:
|
|
2072
|
+
popup_containers.append({
|
|
2073
|
+
'bounds': (x1, y1, x2, y2),
|
|
2074
|
+
'bounds_str': bounds_str,
|
|
2075
|
+
'area': area,
|
|
2076
|
+
'area_ratio': area_ratio,
|
|
2077
|
+
'idx': idx, # 元素在 XML 中的顺序(越后越上层)
|
|
2078
|
+
'class': class_name
|
|
2079
|
+
})
|
|
2080
|
+
|
|
2081
|
+
# 选择最可能的弹窗容器(优先选择:XML 顺序靠后 + 面积适中)
|
|
2082
|
+
if popup_containers:
|
|
2083
|
+
# 按 XML 顺序倒序(后出现的在上层),然后按面积适中程度排序
|
|
2084
|
+
popup_containers.sort(key=lambda x: (x['idx'], -abs(x['area_ratio'] - 0.3)), reverse=True)
|
|
2085
|
+
popup_bounds = popup_containers[0]['bounds']
|
|
2086
|
+
|
|
2087
|
+
# ===== 第二步:在弹窗范围内查找关闭按钮 =====
|
|
2088
|
+
for idx, elem in enumerate(all_elements):
|
|
2089
|
+
text = elem.attrib.get('text', '')
|
|
2090
|
+
content_desc = elem.attrib.get('content-desc', '')
|
|
2091
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2092
|
+
class_name = elem.attrib.get('class', '')
|
|
2093
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2094
|
+
|
|
2095
|
+
if not bounds_str:
|
|
2096
|
+
continue
|
|
2097
|
+
|
|
2098
|
+
# 解析 bounds
|
|
2099
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2100
|
+
if not match:
|
|
2101
|
+
continue
|
|
2102
|
+
|
|
2103
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2104
|
+
width = x2 - x1
|
|
2105
|
+
height = y2 - y1
|
|
2106
|
+
center_x = (x1 + x2) // 2
|
|
2107
|
+
center_y = (y1 + y2) // 2
|
|
2108
|
+
|
|
2109
|
+
# 如果检测到弹窗区域,检查元素是否在弹窗范围内或附近
|
|
2110
|
+
in_popup = True
|
|
2111
|
+
popup_edge_bonus = 0
|
|
2112
|
+
is_floating_close = False # 是否是浮动关闭按钮(在弹窗外部上方)
|
|
2113
|
+
if popup_bounds:
|
|
2114
|
+
px1, py1, px2, py2 = popup_bounds
|
|
2115
|
+
|
|
2116
|
+
# 关闭按钮可能在弹窗外部(常见设计:X 按钮浮在弹窗右上角外侧)
|
|
2117
|
+
# 扩大搜索范围:弹窗上方 200 像素,右侧 50 像素
|
|
2118
|
+
margin_top = 200 # 上方扩展范围(关闭按钮常在弹窗上方)
|
|
2119
|
+
margin_side = 50 # 左右扩展范围
|
|
2120
|
+
margin_bottom = 30 # 下方扩展范围
|
|
2121
|
+
|
|
2122
|
+
in_popup = (px1 - margin_side <= center_x <= px2 + margin_side and
|
|
2123
|
+
py1 - margin_top <= center_y <= py2 + margin_bottom)
|
|
2124
|
+
|
|
2125
|
+
# 检查是否是浮动关闭按钮(在弹窗外侧:上方或下方)
|
|
2126
|
+
# 上方浮动关闭按钮(常见:右上角外侧)
|
|
2127
|
+
if center_y < py1 and center_y > py1 - margin_top:
|
|
2128
|
+
if center_x > (px1 + px2) / 2: # 在弹窗右半部分上方
|
|
2129
|
+
is_floating_close = True
|
|
2130
|
+
# 下方浮动关闭按钮(常见:底部中间外侧)
|
|
2131
|
+
elif center_y > py2 and center_y < py2 + margin_top:
|
|
2132
|
+
# 下方关闭按钮通常在中间位置
|
|
2133
|
+
if abs(center_x - (px1 + px2) / 2) < (px2 - px1) / 2:
|
|
2134
|
+
is_floating_close = True
|
|
2135
|
+
|
|
2136
|
+
if in_popup:
|
|
2137
|
+
# 计算元素是否在弹窗边缘(关闭按钮通常在边缘)
|
|
2138
|
+
dist_to_top = abs(center_y - py1)
|
|
2139
|
+
dist_to_bottom = abs(center_y - py2)
|
|
2140
|
+
dist_to_left = abs(center_x - px1)
|
|
2141
|
+
dist_to_right = abs(center_x - px2)
|
|
2142
|
+
min_dist = min(dist_to_top, dist_to_bottom, dist_to_left, dist_to_right)
|
|
2143
|
+
|
|
2144
|
+
# 在弹窗边缘 100 像素内的元素加分
|
|
2145
|
+
if min_dist < 100:
|
|
2146
|
+
popup_edge_bonus = 3.0 * (1 - min_dist / 100)
|
|
2147
|
+
|
|
2148
|
+
# 浮动关闭按钮(在弹窗上方外侧)给予高额加分
|
|
2149
|
+
if is_floating_close:
|
|
2150
|
+
popup_edge_bonus += 5.0 # 大幅加分
|
|
2151
|
+
|
|
2152
|
+
if not in_popup:
|
|
2153
|
+
continue
|
|
2154
|
+
|
|
2155
|
+
# 相对位置(0-1)
|
|
2156
|
+
rel_x = center_x / screen_width
|
|
2157
|
+
rel_y = center_y / screen_height
|
|
2158
|
+
|
|
2159
|
+
score = 0
|
|
2160
|
+
match_type = ""
|
|
2161
|
+
position = self._get_position_name(rel_x, rel_y)
|
|
2162
|
+
|
|
2163
|
+
# ===== 策略1:精确匹配关闭文本(最高优先级)=====
|
|
2164
|
+
if text in close_texts:
|
|
2165
|
+
score = 15.0 + popup_edge_bonus
|
|
2166
|
+
match_type = f"text='{text}'"
|
|
2167
|
+
|
|
2168
|
+
# ===== 策略2:content-desc 包含关闭关键词 =====
|
|
2169
|
+
elif any(kw in content_desc.lower() for kw in close_desc_keywords):
|
|
2170
|
+
score = 12.0 + popup_edge_bonus
|
|
2171
|
+
match_type = f"desc='{content_desc}'"
|
|
2172
|
+
|
|
2173
|
+
# ===== 策略3:clickable 的小尺寸元素(优先于非 clickable)=====
|
|
2174
|
+
elif clickable:
|
|
2175
|
+
min_size = max(20, int(screen_width * 0.03))
|
|
2176
|
+
max_size = max(120, int(screen_width * 0.15))
|
|
2177
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2178
|
+
# clickable 元素基础分更高
|
|
2179
|
+
base_score = 8.0
|
|
2180
|
+
# 浮动关闭按钮给予最高分
|
|
2181
|
+
if is_floating_close:
|
|
2182
|
+
base_score = 12.0
|
|
2183
|
+
match_type = "floating_close"
|
|
2184
|
+
elif 'Image' in class_name:
|
|
2185
|
+
score = base_score + 2.0
|
|
2186
|
+
match_type = "clickable_image"
|
|
2187
|
+
else:
|
|
2188
|
+
match_type = "clickable"
|
|
2189
|
+
score = base_score + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2190
|
+
|
|
2191
|
+
# ===== 策略4:ImageView/ImageButton 类型的小元素(非 clickable)=====
|
|
2192
|
+
elif 'Image' in class_name:
|
|
2193
|
+
min_size = max(15, int(screen_width * 0.02))
|
|
2194
|
+
max_size = max(120, int(screen_width * 0.12))
|
|
2195
|
+
if min_size <= width <= max_size and min_size <= height <= max_size:
|
|
2196
|
+
score = 5.0 + self._get_position_score(rel_x, rel_y) + popup_edge_bonus
|
|
2197
|
+
match_type = "ImageView"
|
|
2198
|
+
|
|
2199
|
+
# XML 顺序加分(后出现的元素在上层,更可能是弹窗内的元素)
|
|
2200
|
+
if score > 0:
|
|
2201
|
+
xml_order_bonus = idx / len(all_elements) * 2.0 # 最多加 2 分
|
|
2202
|
+
score += xml_order_bonus
|
|
2203
|
+
|
|
2204
|
+
close_candidates.append({
|
|
2205
|
+
'bounds': bounds_str,
|
|
2206
|
+
'center_x': center_x,
|
|
2207
|
+
'center_y': center_y,
|
|
2208
|
+
'width': width,
|
|
2209
|
+
'height': height,
|
|
2210
|
+
'score': score,
|
|
2211
|
+
'position': position,
|
|
2212
|
+
'match_type': match_type,
|
|
2213
|
+
'text': text,
|
|
2214
|
+
'content_desc': content_desc,
|
|
2215
|
+
'x_percent': round(rel_x * 100, 1),
|
|
2216
|
+
'y_percent': round(rel_y * 100, 1),
|
|
2217
|
+
'in_popup': popup_bounds is not None
|
|
2218
|
+
})
|
|
2219
|
+
|
|
2220
|
+
except ET.ParseError:
|
|
2221
|
+
pass
|
|
2222
|
+
|
|
2223
|
+
if not close_candidates:
|
|
2224
|
+
# 如果检测到弹窗区域,先尝试点击常见的关闭按钮位置
|
|
2225
|
+
if popup_bounds:
|
|
2226
|
+
px1, py1, px2, py2 = popup_bounds
|
|
2227
|
+
popup_width = px2 - px1
|
|
2228
|
+
popup_height = py2 - py1
|
|
2229
|
+
|
|
2230
|
+
# 【优化】X按钮有三种常见位置:
|
|
2231
|
+
# 1. 弹窗内靠近顶部边界(内嵌X按钮)- 最常见
|
|
2232
|
+
# 2. 弹窗边界上方(浮动X按钮)
|
|
2233
|
+
# 3. 弹窗正下方(底部关闭按钮)
|
|
2234
|
+
offset_x = max(60, int(popup_width * 0.07)) # 宽度7%
|
|
2235
|
+
offset_y_above = max(35, int(popup_height * 0.025)) # 高度2.5%,在边界之上
|
|
2236
|
+
offset_y_near = max(45, int(popup_height * 0.03)) # 高度3%,紧贴顶边界内侧
|
|
2237
|
+
|
|
2238
|
+
try_positions = [
|
|
2239
|
+
# 【最高优先级】弹窗内紧贴顶部边界
|
|
2240
|
+
(px2 - offset_x, py1 + offset_y_near, "弹窗右上角"),
|
|
2241
|
+
# 弹窗边界上方(浮动X按钮)
|
|
2242
|
+
(px2 - offset_x, py1 - offset_y_above, "弹窗右上浮"),
|
|
2243
|
+
# 弹窗正下方中间(底部关闭按钮)
|
|
2244
|
+
((px1 + px2) // 2, py2 + max(50, int(popup_height * 0.04)), "弹窗下方中间"),
|
|
2245
|
+
# 弹窗正上方中间
|
|
2246
|
+
((px1 + px2) // 2, py1 - 40, "弹窗正上方"),
|
|
2247
|
+
]
|
|
2248
|
+
|
|
2249
|
+
for try_x, try_y, position_name in try_positions:
|
|
2250
|
+
if 0 <= try_x <= screen_width and 0 <= try_y <= screen_height:
|
|
2251
|
+
self.client.u2.click(try_x, try_y)
|
|
2252
|
+
time.sleep(0.3)
|
|
2253
|
+
|
|
2254
|
+
# 尝试后截图,让 AI 判断是否成功
|
|
2255
|
+
screenshot_result = self.take_screenshot("尝试关闭后")
|
|
2256
|
+
return {
|
|
2257
|
+
"success": True,
|
|
2258
|
+
"message": f"✅ 已尝试点击常见关闭按钮位置",
|
|
2259
|
+
"tried_positions": [p[2] for p in try_positions],
|
|
2260
|
+
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2261
|
+
"tip": "请查看截图确认弹窗是否已关闭。如果还在,可手动分析截图找到关闭按钮位置。"
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
# 没有检测到弹窗区域,截图让 AI 分析
|
|
2265
|
+
screenshot_result = self.take_screenshot(description="页面截图", compress=True)
|
|
2266
|
+
|
|
2267
|
+
return {
|
|
2268
|
+
"success": False,
|
|
2269
|
+
"message": "❌ 未检测到弹窗区域,已截图供 AI 分析",
|
|
2270
|
+
"action_required": "请查看截图找到关闭按钮,调用 mobile_click_at_coords 点击",
|
|
2271
|
+
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2272
|
+
"screen_size": {"width": screen_width, "height": screen_height},
|
|
2273
|
+
"image_size": {
|
|
2274
|
+
"width": screenshot_result.get("image_width", screen_width),
|
|
2275
|
+
"height": screenshot_result.get("image_height", screen_height)
|
|
2276
|
+
},
|
|
2277
|
+
"original_size": {
|
|
2278
|
+
"width": screenshot_result.get("original_img_width", screen_width),
|
|
2279
|
+
"height": screenshot_result.get("original_img_height", screen_height)
|
|
2280
|
+
},
|
|
2281
|
+
"search_areas": ["弹窗右上角", "弹窗正上方", "弹窗下方中间", "屏幕右上角"],
|
|
2282
|
+
"time_warning": "⚠️ 截图分析期间弹窗可能自动消失。如果是定时弹窗,建议等待其自动消失。"
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
# 按得分排序,取最可能的
|
|
2286
|
+
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2287
|
+
best = close_candidates[0]
|
|
2288
|
+
|
|
2289
|
+
# 点击
|
|
2290
|
+
self.client.u2.click(best['center_x'], best['center_y'])
|
|
2291
|
+
time.sleep(0.5)
|
|
2292
|
+
|
|
2293
|
+
# 点击后截图,让 AI 判断是否成功
|
|
2294
|
+
screenshot_result = self.take_screenshot("关闭弹窗后")
|
|
2295
|
+
|
|
2296
|
+
# 记录操作(使用百分比,跨设备兼容)
|
|
2297
|
+
self._record_operation(
|
|
2298
|
+
'click',
|
|
2299
|
+
x=best['center_x'],
|
|
2300
|
+
y=best['center_y'],
|
|
2301
|
+
x_percent=best['x_percent'],
|
|
2302
|
+
y_percent=best['y_percent'],
|
|
2303
|
+
screen_width=screen_width,
|
|
2304
|
+
screen_height=screen_height,
|
|
2305
|
+
ref=f"close_popup_{best['position']}"
|
|
2306
|
+
)
|
|
2307
|
+
|
|
2308
|
+
# 返回候选按钮列表,让 AI 看截图判断
|
|
2309
|
+
# 如果弹窗还在,AI 可以选择点击其他候选按钮
|
|
2310
|
+
return {
|
|
2311
|
+
"success": True,
|
|
2312
|
+
"message": f"✅ 已点击关闭按钮 ({best['position']}): ({best['center_x']}, {best['center_y']})",
|
|
2313
|
+
"clicked": {
|
|
2314
|
+
"position": best['position'],
|
|
2315
|
+
"match_type": best['match_type'],
|
|
2316
|
+
"coords": (best['center_x'], best['center_y']),
|
|
2317
|
+
"percent": (best['x_percent'], best['y_percent'])
|
|
2318
|
+
},
|
|
2319
|
+
"screenshot": screenshot_result.get("screenshot_path", ""),
|
|
2320
|
+
"popup_detected": popup_bounds is not None,
|
|
2321
|
+
"popup_bounds": f"[{popup_bounds[0]},{popup_bounds[1]}][{popup_bounds[2]},{popup_bounds[3]}]" if popup_bounds else None,
|
|
2322
|
+
"other_candidates": [
|
|
2323
|
+
{
|
|
2324
|
+
"position": c['position'],
|
|
2325
|
+
"type": c['match_type'],
|
|
2326
|
+
"coords": (c['center_x'], c['center_y']),
|
|
2327
|
+
"percent": (c['x_percent'], c['y_percent'])
|
|
2328
|
+
}
|
|
2329
|
+
for c in close_candidates[1:4] # 返回其他3个候选,AI 可以选择
|
|
2330
|
+
],
|
|
2331
|
+
"tip": "请查看截图判断弹窗是否已关闭。如果弹窗还在,可以尝试点击 other_candidates 中的其他位置;如果误点跳转了,请按返回键"
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
except Exception as e:
|
|
2335
|
+
return {"success": False, "message": f"❌ 关闭弹窗失败: {e}"}
|
|
2336
|
+
|
|
2337
|
+
def _get_position_name(self, rel_x: float, rel_y: float) -> str:
|
|
2338
|
+
"""根据相对坐标获取位置名称"""
|
|
2339
|
+
if rel_y < 0.4:
|
|
2340
|
+
if rel_x > 0.6:
|
|
2341
|
+
return "右上角"
|
|
2342
|
+
elif rel_x < 0.4:
|
|
2343
|
+
return "左上角"
|
|
2344
|
+
else:
|
|
2345
|
+
return "顶部中间"
|
|
2346
|
+
elif rel_y > 0.6:
|
|
2347
|
+
if rel_x > 0.6:
|
|
2348
|
+
return "右下角"
|
|
2349
|
+
elif rel_x < 0.4:
|
|
2350
|
+
return "左下角"
|
|
2351
|
+
else:
|
|
2352
|
+
return "底部中间"
|
|
2353
|
+
else:
|
|
2354
|
+
if rel_x > 0.6:
|
|
2355
|
+
return "右侧"
|
|
2356
|
+
elif rel_x < 0.4:
|
|
2357
|
+
return "左侧"
|
|
2358
|
+
else:
|
|
2359
|
+
return "中间"
|
|
2360
|
+
|
|
2361
|
+
def _get_position_score(self, rel_x: float, rel_y: float) -> float:
|
|
2362
|
+
"""根据位置计算额外得分(角落位置加分更多)"""
|
|
2363
|
+
# 弹窗关闭按钮常见位置得分:右上角 > 左上角 > 底部中间 > 其他角落
|
|
2364
|
+
if rel_y < 0.4: # 上半部分
|
|
2365
|
+
if rel_x > 0.6: # 右上角
|
|
2366
|
+
return 2.0 + (rel_x - 0.6) + (0.4 - rel_y)
|
|
2367
|
+
elif rel_x < 0.4: # 左上角
|
|
2368
|
+
return 1.5 + (0.4 - rel_x) + (0.4 - rel_y)
|
|
2369
|
+
else: # 顶部中间
|
|
2370
|
+
return 1.0
|
|
2371
|
+
elif rel_y > 0.6: # 下半部分
|
|
2372
|
+
if 0.3 < rel_x < 0.7: # 底部中间
|
|
2373
|
+
return 1.2 + (1 - abs(rel_x - 0.5) * 2)
|
|
2374
|
+
else: # 底部角落
|
|
2375
|
+
return 0.8
|
|
2376
|
+
else: # 中间区域
|
|
2377
|
+
return 0.5
|
|
2378
|
+
|
|
2379
|
+
def assert_text(self, text: str) -> Dict:
|
|
2380
|
+
"""检查页面是否包含文本(支持精确匹配和包含匹配)"""
|
|
2381
|
+
try:
|
|
2382
|
+
exists = False
|
|
2383
|
+
match_type = ""
|
|
2384
|
+
|
|
2385
|
+
if self._is_ios():
|
|
2386
|
+
ios_client = self._get_ios_client()
|
|
2387
|
+
if ios_client and hasattr(ios_client, 'wda'):
|
|
2388
|
+
# 先尝试精确匹配
|
|
2389
|
+
if ios_client.wda(name=text).exists or ios_client.wda(label=text).exists:
|
|
2390
|
+
exists = True
|
|
2391
|
+
match_type = "精确匹配"
|
|
2392
|
+
# 再尝试包含匹配
|
|
2393
|
+
elif ios_client.wda(nameContains=text).exists or ios_client.wda(labelContains=text).exists:
|
|
2394
|
+
exists = True
|
|
2395
|
+
match_type = "包含匹配"
|
|
2396
|
+
else:
|
|
2397
|
+
# Android: 先尝试精确匹配
|
|
2398
|
+
if self.client.u2(text=text).exists():
|
|
2399
|
+
exists = True
|
|
2400
|
+
match_type = "精确匹配"
|
|
2401
|
+
# 再尝试包含匹配
|
|
2402
|
+
elif self.client.u2(textContains=text).exists():
|
|
2403
|
+
exists = True
|
|
2404
|
+
match_type = "包含匹配"
|
|
2405
|
+
|
|
2406
|
+
if exists:
|
|
2407
|
+
message = f"✅ 文本'{text}' 存在({match_type})"
|
|
2408
|
+
else:
|
|
2409
|
+
message = f"❌ 文本'{text}' 不存在"
|
|
2410
|
+
|
|
2411
|
+
return {
|
|
2412
|
+
"success": True,
|
|
2413
|
+
"found": exists,
|
|
2414
|
+
"text": text,
|
|
2415
|
+
"match_type": match_type if exists else None,
|
|
2416
|
+
"message": message
|
|
2417
|
+
}
|
|
2418
|
+
except Exception as e:
|
|
2419
|
+
return {"success": False, "message": f"❌ 断言失败: {e}"}
|
|
2420
|
+
|
|
2421
|
+
# ==================== 脚本生成 ====================
|
|
2422
|
+
|
|
2423
|
+
def get_operation_history(self, limit: Optional[int] = None) -> Dict:
|
|
2424
|
+
"""获取操作历史"""
|
|
2425
|
+
history = self.operation_history
|
|
2426
|
+
if limit:
|
|
2427
|
+
history = history[-limit:]
|
|
2428
|
+
return {
|
|
2429
|
+
"success": True,
|
|
2430
|
+
"count": len(history),
|
|
2431
|
+
"total": len(self.operation_history),
|
|
2432
|
+
"operations": history
|
|
2433
|
+
}
|
|
2434
|
+
|
|
2435
|
+
def clear_operation_history(self) -> Dict:
|
|
2436
|
+
"""清空操作历史"""
|
|
2437
|
+
count = len(self.operation_history)
|
|
2438
|
+
self.operation_history = []
|
|
2439
|
+
return {"success": True, "message": f"✅ 已清空 {count} 条记录"}
|
|
2440
|
+
|
|
2441
|
+
def generate_test_script(self, test_name: str, package_name: str, filename: str) -> Dict:
|
|
2442
|
+
"""生成 pytest 测试脚本(带智能等待、广告处理和跨设备兼容)
|
|
2443
|
+
|
|
2444
|
+
优化:
|
|
2445
|
+
1. 坐标点击自动转换为百分比定位(跨分辨率兼容)
|
|
2446
|
+
2. 优先使用 ID/文本定位(最稳定)
|
|
2447
|
+
3. 百分比定位作为坐标的替代方案
|
|
2448
|
+
"""
|
|
2449
|
+
if not self.operation_history:
|
|
2450
|
+
return {"success": False, "message": "❌ 没有操作历史,请先执行一些操作"}
|
|
2451
|
+
|
|
2452
|
+
# 生成脚本
|
|
2453
|
+
safe_name = re.sub(r'[^\w\s-]', '', test_name).strip().replace(' ', '_')
|
|
2454
|
+
|
|
2455
|
+
script_lines = [
|
|
2456
|
+
"#!/usr/bin/env python3",
|
|
2457
|
+
"# -*- coding: utf-8 -*-",
|
|
2458
|
+
f'"""',
|
|
2459
|
+
f"测试用例: {test_name}",
|
|
2460
|
+
f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
|
2461
|
+
"",
|
|
2462
|
+
"定位策略(按优先级):",
|
|
2463
|
+
"1. ID 定位 - 最稳定,跨设备兼容",
|
|
2464
|
+
"2. 文本定位 - 稳定,跨设备兼容",
|
|
2465
|
+
"3. 百分比定位 - 跨分辨率兼容(坐标自动转换)",
|
|
2466
|
+
f'"""',
|
|
2467
|
+
"import time",
|
|
2468
|
+
"import uiautomator2 as u2",
|
|
2469
|
+
"",
|
|
2470
|
+
f'PACKAGE_NAME = "{package_name}"',
|
|
2471
|
+
"",
|
|
2472
|
+
"# === 配置(根据 App 情况调整)===",
|
|
2473
|
+
"LAUNCH_WAIT = 3 # 启动后等待时间(秒)",
|
|
2474
|
+
"CLOSE_AD_ON_LAUNCH = True # 是否尝试关闭启动广告",
|
|
2475
|
+
"AD_CLOSE_KEYWORDS = ['关闭', '跳过', 'Skip', 'Close', '×', 'X', '我知道了', '稍后再说']",
|
|
2476
|
+
"",
|
|
2477
|
+
"",
|
|
2478
|
+
"def smart_wait(d, seconds=1):",
|
|
2479
|
+
' """等待页面稳定"""',
|
|
2480
|
+
" time.sleep(seconds)",
|
|
2481
|
+
"",
|
|
2482
|
+
"",
|
|
2483
|
+
"def close_ad_if_exists(d, quick=False):",
|
|
2484
|
+
' """尝试关闭广告弹窗(quick=True 时只检查常见的)"""',
|
|
2485
|
+
" keywords = AD_CLOSE_KEYWORDS[:3] if quick else AD_CLOSE_KEYWORDS",
|
|
2486
|
+
" for keyword in keywords:",
|
|
2487
|
+
" elem = d(textContains=keyword)",
|
|
2488
|
+
" if elem.exists(timeout=0.3): # 缩短超时",
|
|
2489
|
+
" try:",
|
|
2490
|
+
" elem.click()",
|
|
2491
|
+
" print(f' 📢 关闭广告: {keyword}')",
|
|
2492
|
+
" time.sleep(0.3)",
|
|
2493
|
+
" return True",
|
|
2494
|
+
" except:",
|
|
2495
|
+
" pass",
|
|
2496
|
+
" return False",
|
|
2497
|
+
"",
|
|
2498
|
+
"",
|
|
2499
|
+
"def safe_click(d, selector, timeout=3):",
|
|
2500
|
+
' """安全点击(带等待)"""',
|
|
2501
|
+
" try:",
|
|
2502
|
+
" if selector.exists(timeout=timeout):",
|
|
2503
|
+
" selector.click()",
|
|
2504
|
+
" return True",
|
|
2505
|
+
" return False",
|
|
2506
|
+
" except Exception as e:",
|
|
2507
|
+
" print(f' ⚠️ 点击失败: {e}')",
|
|
2508
|
+
" return False",
|
|
2509
|
+
"",
|
|
2510
|
+
"",
|
|
2511
|
+
"def click_by_percent(d, x_percent, y_percent):",
|
|
2512
|
+
' """',
|
|
2513
|
+
' 百分比点击(跨分辨率兼容)',
|
|
2514
|
+
' ',
|
|
2515
|
+
' 原理:屏幕左上角 (0%, 0%),右下角 (100%, 100%)',
|
|
2516
|
+
' 优势:同样的百分比在不同分辨率设备上都能点到相同相对位置',
|
|
2517
|
+
' """',
|
|
2518
|
+
" info = d.info",
|
|
2519
|
+
" width = info.get('displayWidth', 0)",
|
|
2520
|
+
" height = info.get('displayHeight', 0)",
|
|
2521
|
+
" x = int(width * x_percent / 100)",
|
|
2522
|
+
" y = int(height * y_percent / 100)",
|
|
2523
|
+
" d.click(x, y)",
|
|
2524
|
+
" return True",
|
|
2525
|
+
"",
|
|
2526
|
+
"",
|
|
2527
|
+
"def long_press_by_percent(d, x_percent, y_percent, duration=1.0):",
|
|
2528
|
+
' """',
|
|
2529
|
+
' 百分比长按(跨分辨率兼容)',
|
|
2530
|
+
' ',
|
|
2531
|
+
' 原理:屏幕左上角 (0%, 0%),右下角 (100%, 100%)',
|
|
2532
|
+
' 优势:同样的百分比在不同分辨率设备上都能长按到相同相对位置',
|
|
2533
|
+
' """',
|
|
2534
|
+
" info = d.info",
|
|
2535
|
+
" width = info.get('displayWidth', 0)",
|
|
2536
|
+
" height = info.get('displayHeight', 0)",
|
|
2537
|
+
" x = int(width * x_percent / 100)",
|
|
2538
|
+
" y = int(height * y_percent / 100)",
|
|
2539
|
+
" d.long_click(x, y, duration=duration)",
|
|
2540
|
+
" return True",
|
|
2541
|
+
"",
|
|
2542
|
+
"",
|
|
2543
|
+
"def test_main():",
|
|
2544
|
+
" # 连接设备",
|
|
2545
|
+
" d = u2.connect()",
|
|
2546
|
+
" d.implicitly_wait(10) # 设置全局等待",
|
|
2547
|
+
" ",
|
|
2548
|
+
" # 启动应用",
|
|
2549
|
+
f" d.app_start(PACKAGE_NAME)",
|
|
2550
|
+
" time.sleep(LAUNCH_WAIT) # 等待启动(可调整)",
|
|
2551
|
+
" ",
|
|
2552
|
+
" # 尝试关闭启动广告(可选,根据 App 情况调整)",
|
|
2553
|
+
" if CLOSE_AD_ON_LAUNCH:",
|
|
2554
|
+
" close_ad_if_exists(d)",
|
|
2555
|
+
" ",
|
|
2556
|
+
]
|
|
2557
|
+
|
|
2558
|
+
# 生成操作代码(跳过启动应用相关操作,因为脚本头部已处理)
|
|
2559
|
+
step_num = 0
|
|
2560
|
+
for op in self.operation_history:
|
|
2561
|
+
action = op.get('action')
|
|
2562
|
+
|
|
2563
|
+
# 跳过 launch_app(脚本头部已经有 app_start)
|
|
2564
|
+
if action == 'launch_app':
|
|
2565
|
+
continue
|
|
2566
|
+
|
|
2567
|
+
step_num += 1
|
|
2568
|
+
|
|
2569
|
+
if action == 'click':
|
|
2570
|
+
ref = op.get('ref', '')
|
|
2571
|
+
element = op.get('element', '')
|
|
2572
|
+
has_coords = 'x' in op and 'y' in op
|
|
2573
|
+
has_percent = 'x_percent' in op and 'y_percent' in op
|
|
2574
|
+
|
|
2575
|
+
# 判断 ref 是否为坐标格式(coords_ 或 coords:)
|
|
2576
|
+
is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
|
|
2577
|
+
is_percent_ref = ref.startswith('percent_')
|
|
2578
|
+
|
|
2579
|
+
# 优先级:ID > 文本 > 百分比 > 坐标(兜底)
|
|
2580
|
+
if ref and (':id/' in ref or ref.startswith('com.')):
|
|
2581
|
+
# 1️⃣ 使用 resource-id(最稳定)
|
|
2582
|
+
script_lines.append(f" # 步骤{step_num}: 点击元素 (ID定位,最稳定)")
|
|
2583
|
+
script_lines.append(f" safe_click(d, d(resourceId='{ref}'))")
|
|
2584
|
+
elif ref and not is_coords_ref and not is_percent_ref and ':' not in ref:
|
|
2585
|
+
# 2️⃣ 使用文本(稳定)- 排除 "text:xxx" 等带冒号的格式
|
|
2586
|
+
script_lines.append(f" # 步骤{step_num}: 点击文本 '{ref}' (文本定位)")
|
|
2587
|
+
script_lines.append(f" safe_click(d, d(text='{ref}'))")
|
|
2588
|
+
elif ref and ':' in ref and not is_coords_ref and not is_percent_ref:
|
|
2589
|
+
# 2️⃣-b 使用文本(Android 的 text:xxx 或 description:xxx 格式)
|
|
2590
|
+
# 提取冒号后面的实际文本值
|
|
2591
|
+
actual_text = ref.split(':', 1)[1] if ':' in ref else ref
|
|
2592
|
+
script_lines.append(f" # 步骤{step_num}: 点击文本 '{actual_text}' (文本定位)")
|
|
2593
|
+
script_lines.append(f" safe_click(d, d(text='{actual_text}'))")
|
|
2594
|
+
elif has_percent:
|
|
2595
|
+
# 3️⃣ 使用百分比(跨分辨率兼容)
|
|
2596
|
+
x_pct = op['x_percent']
|
|
2597
|
+
y_pct = op['y_percent']
|
|
2598
|
+
desc = f" ({element})" if element else ""
|
|
2599
|
+
script_lines.append(f" # 步骤{step_num}: 点击位置{desc} (百分比定位,跨分辨率兼容)")
|
|
2600
|
+
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
2601
|
+
elif has_coords:
|
|
2602
|
+
# 4️⃣ 坐标兜底(不推荐,仅用于无法获取百分比的情况)
|
|
2603
|
+
desc = f" ({element})" if element else ""
|
|
2604
|
+
script_lines.append(f" # 步骤{step_num}: 点击坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
2605
|
+
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
2606
|
+
else:
|
|
2607
|
+
continue # 无效操作,跳过
|
|
2608
|
+
|
|
2609
|
+
script_lines.append(" time.sleep(0.5) # 等待响应")
|
|
2610
|
+
script_lines.append(" ")
|
|
2611
|
+
|
|
2612
|
+
elif action == 'input':
|
|
2613
|
+
text = op.get('text', '')
|
|
2614
|
+
ref = op.get('ref', '')
|
|
2615
|
+
has_coords = 'x' in op and 'y' in op
|
|
2616
|
+
has_percent = 'x_percent' in op and 'y_percent' in op
|
|
2617
|
+
|
|
2618
|
+
# 判断 ref 是否为坐标格式
|
|
2619
|
+
is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
|
|
2620
|
+
|
|
2621
|
+
# 优先使用 ID,其次百分比,最后坐标
|
|
2622
|
+
if ref and not is_coords_ref and (':id/' in ref or ref.startswith('com.')):
|
|
2623
|
+
# 完整格式的 resource-id
|
|
2624
|
+
script_lines.append(f" # 步骤{step_num}: 输入文本 '{text}' (ID定位)")
|
|
2625
|
+
script_lines.append(f" d(resourceId='{ref}').set_text('{text}')")
|
|
2626
|
+
elif ref and not is_coords_ref and not has_coords:
|
|
2627
|
+
# 简短格式的 resource-id(不包含 com. 或 :id/)
|
|
2628
|
+
script_lines.append(f" # 步骤{step_num}: 输入文本 '{text}' (ID定位)")
|
|
2629
|
+
script_lines.append(f" d(resourceId='{ref}').set_text('{text}')")
|
|
2630
|
+
elif has_percent:
|
|
2631
|
+
x_pct = op['x_percent']
|
|
2632
|
+
y_pct = op['y_percent']
|
|
2633
|
+
script_lines.append(f" # 步骤{step_num}: 点击后输入 (百分比定位)")
|
|
2634
|
+
script_lines.append(f" click_by_percent(d, {x_pct}, {y_pct})")
|
|
2635
|
+
script_lines.append(f" time.sleep(0.3)")
|
|
2636
|
+
script_lines.append(f" d.send_keys('{text}')")
|
|
2637
|
+
elif has_coords:
|
|
2638
|
+
script_lines.append(f" # 步骤{step_num}: 点击坐标后输入 (⚠️ 可能不兼容其他分辨率)")
|
|
2639
|
+
script_lines.append(f" d.click({op['x']}, {op['y']})")
|
|
2640
|
+
script_lines.append(f" time.sleep(0.3)")
|
|
2641
|
+
script_lines.append(f" d.send_keys('{text}')")
|
|
2642
|
+
else:
|
|
2643
|
+
# 兜底:无法识别的格式,跳过
|
|
2644
|
+
continue
|
|
2645
|
+
script_lines.append(" time.sleep(0.5)")
|
|
2646
|
+
script_lines.append(" ")
|
|
2647
|
+
|
|
2648
|
+
elif action == 'long_press':
|
|
2649
|
+
ref = op.get('ref', '')
|
|
2650
|
+
element = op.get('element', '')
|
|
2651
|
+
duration = op.get('duration', 1.0)
|
|
2652
|
+
has_coords = 'x' in op and 'y' in op
|
|
2653
|
+
has_percent = 'x_percent' in op and 'y_percent' in op
|
|
2654
|
+
|
|
2655
|
+
# 判断 ref 是否为坐标格式
|
|
2656
|
+
is_coords_ref = ref.startswith('coords_') or ref.startswith('coords:')
|
|
2657
|
+
is_percent_ref = ref.startswith('percent_')
|
|
2658
|
+
|
|
2659
|
+
# 优先级:ID > 文本 > 百分比 > 坐标
|
|
2660
|
+
if ref and (':id/' in ref or ref.startswith('com.')):
|
|
2661
|
+
# 使用 resource-id
|
|
2662
|
+
script_lines.append(f" # 步骤{step_num}: 长按元素 (ID定位,最稳定)")
|
|
2663
|
+
script_lines.append(f" d(resourceId='{ref}').long_click(duration={duration})")
|
|
2664
|
+
elif ref and not is_coords_ref and not is_percent_ref and ':' not in ref:
|
|
2665
|
+
# 使用文本
|
|
2666
|
+
script_lines.append(f" # 步骤{step_num}: 长按文本 '{ref}' (文本定位)")
|
|
2667
|
+
script_lines.append(f" d(text='{ref}').long_click(duration={duration})")
|
|
2668
|
+
elif ref and ':' in ref and not is_coords_ref and not is_percent_ref:
|
|
2669
|
+
actual_text = ref.split(':', 1)[1] if ':' in ref else ref
|
|
2670
|
+
script_lines.append(f" # 步骤{step_num}: 长按文本 '{actual_text}' (文本定位)")
|
|
2671
|
+
script_lines.append(f" d(text='{actual_text}').long_click(duration={duration})")
|
|
2672
|
+
elif has_percent:
|
|
2673
|
+
# 使用百分比
|
|
2674
|
+
x_pct = op['x_percent']
|
|
2675
|
+
y_pct = op['y_percent']
|
|
2676
|
+
desc = f" ({element})" if element else ""
|
|
2677
|
+
script_lines.append(f" # 步骤{step_num}: 长按位置{desc} (百分比定位,跨分辨率兼容)")
|
|
2678
|
+
script_lines.append(f" long_press_by_percent(d, {x_pct}, {y_pct}, duration={duration}) # 原坐标: ({op.get('x', '?')}, {op.get('y', '?')})")
|
|
2679
|
+
elif has_coords:
|
|
2680
|
+
# 坐标兜底
|
|
2681
|
+
desc = f" ({element})" if element else ""
|
|
2682
|
+
script_lines.append(f" # 步骤{step_num}: 长按坐标{desc} (⚠️ 坐标定位,可能不兼容其他分辨率)")
|
|
2683
|
+
script_lines.append(f" d.long_click({op['x']}, {op['y']}, duration={duration})")
|
|
2684
|
+
else:
|
|
2685
|
+
continue
|
|
2686
|
+
|
|
2687
|
+
script_lines.append(" time.sleep(0.5) # 等待响应")
|
|
2688
|
+
script_lines.append(" ")
|
|
2689
|
+
|
|
2690
|
+
elif action == 'swipe':
|
|
2691
|
+
direction = op.get('direction', 'up')
|
|
2692
|
+
script_lines.append(f" # 步骤{step_num}: 滑动 {direction}")
|
|
2693
|
+
script_lines.append(f" d.swipe_ext('{direction}')")
|
|
2694
|
+
script_lines.append(" time.sleep(0.5)")
|
|
2695
|
+
script_lines.append(" ")
|
|
2696
|
+
|
|
2697
|
+
elif action == 'press_key':
|
|
2698
|
+
key = op.get('key', 'enter')
|
|
2699
|
+
script_lines.append(f" # 步骤{step_num}: 按键 {key}")
|
|
2700
|
+
script_lines.append(f" d.press('{key}')")
|
|
2701
|
+
script_lines.append(" time.sleep(0.5)")
|
|
2702
|
+
script_lines.append(" ")
|
|
2703
|
+
|
|
2704
|
+
script_lines.extend([
|
|
2705
|
+
" print('✅ 测试完成')",
|
|
2706
|
+
"",
|
|
2707
|
+
"",
|
|
2708
|
+
"if __name__ == '__main__':",
|
|
2709
|
+
" test_main()",
|
|
2710
|
+
])
|
|
2711
|
+
|
|
2712
|
+
script = '\n'.join(script_lines)
|
|
2713
|
+
|
|
2714
|
+
# 保存文件
|
|
2715
|
+
output_dir = Path("tests")
|
|
2716
|
+
output_dir.mkdir(exist_ok=True)
|
|
2717
|
+
|
|
2718
|
+
if not filename.endswith('.py'):
|
|
2719
|
+
filename = f"{filename}.py"
|
|
2720
|
+
|
|
2721
|
+
file_path = output_dir / filename
|
|
2722
|
+
file_path.write_text(script, encoding='utf-8')
|
|
2723
|
+
|
|
2724
|
+
return {
|
|
2725
|
+
"success": True,
|
|
2726
|
+
"file_path": str(file_path),
|
|
2727
|
+
"message": f"✅ 脚本已生成: {file_path}",
|
|
2728
|
+
"operations_count": len(self.operation_history),
|
|
2729
|
+
"preview": script[:500] + "..."
|
|
2730
|
+
}
|
|
2731
|
+
|
|
2732
|
+
# ========== 模板匹配功能 ==========
|
|
2733
|
+
|
|
2734
|
+
def template_match_close(self, screenshot_path: Optional[str] = None, threshold: float = 0.75) -> Dict:
|
|
2735
|
+
"""使用模板匹配查找关闭按钮
|
|
2736
|
+
|
|
2737
|
+
基于 OpenCV 模板匹配,从预设的X号模板库中查找匹配项。
|
|
2738
|
+
比 AI 视觉识别更精准、更快速。
|
|
2739
|
+
|
|
2740
|
+
Args:
|
|
2741
|
+
screenshot_path: 截图路径(可选,不提供则自动截图)
|
|
2742
|
+
threshold: 匹配阈值 0-1,越高越严格,默认0.75
|
|
2743
|
+
|
|
2744
|
+
Returns:
|
|
2745
|
+
匹配结果,包含坐标和点击命令
|
|
2746
|
+
"""
|
|
2747
|
+
try:
|
|
2748
|
+
from .template_matcher import TemplateMatcher
|
|
2749
|
+
|
|
2750
|
+
# 如果没有提供截图,先截图
|
|
2751
|
+
if screenshot_path is None:
|
|
2752
|
+
screenshot_result = self.take_screenshot(description="模板匹配", compress=False)
|
|
2753
|
+
screenshot_path = screenshot_result.get("screenshot_path")
|
|
2754
|
+
if not screenshot_path:
|
|
2755
|
+
return {"success": False, "error": "截图失败"}
|
|
2756
|
+
|
|
2757
|
+
matcher = TemplateMatcher()
|
|
2758
|
+
result = matcher.find_close_buttons(screenshot_path, threshold)
|
|
2759
|
+
|
|
2760
|
+
return result
|
|
2761
|
+
|
|
2762
|
+
except ImportError:
|
|
2763
|
+
return {
|
|
2764
|
+
"success": False,
|
|
2765
|
+
"error": "需要安装 opencv-python: pip install opencv-python"
|
|
2766
|
+
}
|
|
2767
|
+
except Exception as e:
|
|
2768
|
+
return {"success": False, "error": f"模板匹配失败: {e}"}
|
|
2769
|
+
|
|
2770
|
+
def template_click_close(self, threshold: float = 0.75) -> Dict:
|
|
2771
|
+
"""模板匹配并点击关闭按钮(一步到位)
|
|
2772
|
+
|
|
2773
|
+
截图 -> 模板匹配 -> 点击最佳匹配位置
|
|
2774
|
+
|
|
2775
|
+
Args:
|
|
2776
|
+
threshold: 匹配阈值 0-1
|
|
2777
|
+
|
|
2778
|
+
Returns:
|
|
2779
|
+
操作结果
|
|
2780
|
+
"""
|
|
2781
|
+
try:
|
|
2782
|
+
# 先截图并匹配
|
|
2783
|
+
match_result = self.template_match_close(threshold=threshold)
|
|
2784
|
+
|
|
2785
|
+
if not match_result.get("success"):
|
|
2786
|
+
return match_result
|
|
2787
|
+
|
|
2788
|
+
# 获取最佳匹配的百分比坐标
|
|
2789
|
+
best = match_result.get("best_match", {})
|
|
2790
|
+
x_percent = best.get("percent", {}).get("x")
|
|
2791
|
+
y_percent = best.get("percent", {}).get("y")
|
|
2792
|
+
|
|
2793
|
+
if x_percent is None or y_percent is None:
|
|
2794
|
+
return {"success": False, "error": "无法获取匹配坐标"}
|
|
2795
|
+
|
|
2796
|
+
# 点击
|
|
2797
|
+
click_result = self.click_by_percent(x_percent, y_percent)
|
|
2798
|
+
|
|
2799
|
+
return {
|
|
2800
|
+
"success": True,
|
|
2801
|
+
"message": f"✅ 模板匹配并点击成功",
|
|
2802
|
+
"matched_template": best.get("template"),
|
|
2803
|
+
"confidence": best.get("confidence"),
|
|
2804
|
+
"clicked_position": f"({x_percent}%, {y_percent}%)",
|
|
2805
|
+
"click_result": click_result
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
except Exception as e:
|
|
2809
|
+
return {"success": False, "error": f"模板点击失败: {e}"}
|
|
2810
|
+
|
|
2811
|
+
def template_add(self, screenshot_path: str, x: int, y: int,
|
|
2812
|
+
width: int, height: int, template_name: str) -> Dict:
|
|
2813
|
+
"""从截图中裁剪并添加新模板
|
|
2814
|
+
|
|
2815
|
+
当遇到新样式的X号时,用此方法添加到模板库。
|
|
2816
|
+
|
|
2817
|
+
Args:
|
|
2818
|
+
screenshot_path: 截图路径
|
|
2819
|
+
x, y: 裁剪区域左上角坐标
|
|
2820
|
+
width, height: 裁剪区域大小
|
|
2821
|
+
template_name: 模板名称(如 x_circle_gray)
|
|
2822
|
+
|
|
2823
|
+
Returns:
|
|
2824
|
+
结果
|
|
2825
|
+
"""
|
|
2826
|
+
try:
|
|
2827
|
+
from .template_matcher import TemplateMatcher
|
|
2828
|
+
|
|
2829
|
+
matcher = TemplateMatcher()
|
|
2830
|
+
return matcher.crop_and_add_template(
|
|
2831
|
+
screenshot_path, x, y, width, height, template_name
|
|
2832
|
+
)
|
|
2833
|
+
except ImportError:
|
|
2834
|
+
return {"success": False, "error": "需要安装 opencv-python"}
|
|
2835
|
+
except Exception as e:
|
|
2836
|
+
return {"success": False, "error": f"添加模板失败: {e}"}
|
|
2837
|
+
|
|
2838
|
+
def template_list(self) -> Dict:
|
|
2839
|
+
"""列出所有关闭按钮模板"""
|
|
2840
|
+
try:
|
|
2841
|
+
from .template_matcher import TemplateMatcher
|
|
2842
|
+
|
|
2843
|
+
matcher = TemplateMatcher()
|
|
2844
|
+
return matcher.list_templates()
|
|
2845
|
+
except ImportError:
|
|
2846
|
+
return {"success": False, "error": "需要安装 opencv-python"}
|
|
2847
|
+
except Exception as e:
|
|
2848
|
+
return {"success": False, "error": f"列出模板失败: {e}"}
|
|
2849
|
+
|
|
2850
|
+
def template_delete(self, template_name: str) -> Dict:
|
|
2851
|
+
"""删除指定模板"""
|
|
2852
|
+
try:
|
|
2853
|
+
from .template_matcher import TemplateMatcher
|
|
2854
|
+
|
|
2855
|
+
matcher = TemplateMatcher()
|
|
2856
|
+
return matcher.delete_template(template_name)
|
|
2857
|
+
except ImportError:
|
|
2858
|
+
return {"success": False, "error": "需要安装 opencv-python"}
|
|
2859
|
+
except Exception as e:
|
|
2860
|
+
return {"success": False, "error": f"删除模板失败: {e}"}
|
|
2861
|
+
|
|
2862
|
+
def close_ad_popup(self, auto_learn: bool = True) -> Dict:
|
|
2863
|
+
"""智能关闭广告弹窗(专用于广告场景)
|
|
2864
|
+
|
|
2865
|
+
按优先级尝试:
|
|
2866
|
+
1. 控件树查找关闭按钮(最可靠)
|
|
2867
|
+
2. 模板匹配(需要积累模板库)
|
|
2868
|
+
3. 返回视觉信息供 AI 分析(如果前两步失败)
|
|
2869
|
+
|
|
2870
|
+
自动学习:
|
|
2871
|
+
- 点击成功后,检查这个 X 是否已在模板库
|
|
2872
|
+
- 如果是新样式,自动裁剪并添加到模板库
|
|
2873
|
+
|
|
2874
|
+
Args:
|
|
2875
|
+
auto_learn: 是否自动学习新模板(点击成功后检查并保存)
|
|
2876
|
+
|
|
2877
|
+
Returns:
|
|
2878
|
+
结果字典
|
|
2879
|
+
"""
|
|
2880
|
+
import time
|
|
2881
|
+
import re
|
|
2882
|
+
|
|
2883
|
+
result = {
|
|
2884
|
+
"success": False,
|
|
2885
|
+
"method": None,
|
|
2886
|
+
"message": "",
|
|
2887
|
+
"learned_template": None
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
if self._is_ios():
|
|
2891
|
+
return {"success": False, "error": "iOS 暂不支持此功能"}
|
|
2892
|
+
|
|
2893
|
+
try:
|
|
2894
|
+
import xml.etree.ElementTree as ET
|
|
2895
|
+
|
|
2896
|
+
# ========== 第1步:控件树查找关闭按钮 ==========
|
|
2897
|
+
xml_string = self.client.u2.dump_hierarchy(compressed=False)
|
|
2898
|
+
root = ET.fromstring(xml_string)
|
|
2899
|
+
|
|
2900
|
+
# 关闭按钮的常见特征
|
|
2901
|
+
close_keywords = ['关闭', '跳过', '×', 'X', 'x', 'close', 'skip', '取消']
|
|
2902
|
+
close_content_desc = ['关闭', '跳过', 'close', 'skip', 'dismiss']
|
|
2903
|
+
|
|
2904
|
+
close_candidates = []
|
|
2905
|
+
|
|
2906
|
+
for elem in root.iter():
|
|
2907
|
+
text = elem.attrib.get('text', '').strip()
|
|
2908
|
+
content_desc = elem.attrib.get('content-desc', '').strip()
|
|
2909
|
+
clickable = elem.attrib.get('clickable', 'false') == 'true'
|
|
2910
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
2911
|
+
resource_id = elem.attrib.get('resource-id', '')
|
|
2912
|
+
|
|
2913
|
+
if not bounds_str:
|
|
2914
|
+
continue
|
|
2915
|
+
|
|
2916
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
2917
|
+
if not match:
|
|
2918
|
+
continue
|
|
2919
|
+
|
|
2920
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
2921
|
+
width = x2 - x1
|
|
2922
|
+
height = y2 - y1
|
|
2923
|
+
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
|
|
2924
|
+
|
|
2925
|
+
score = 0
|
|
2926
|
+
reason = ""
|
|
2927
|
+
|
|
2928
|
+
# 文本匹配
|
|
2929
|
+
for kw in close_keywords:
|
|
2930
|
+
if kw in text:
|
|
2931
|
+
score += 10
|
|
2932
|
+
reason = f"文本含'{kw}'"
|
|
2933
|
+
break
|
|
2934
|
+
|
|
2935
|
+
# content-desc 匹配
|
|
2936
|
+
for kw in close_content_desc:
|
|
2937
|
+
if kw.lower() in content_desc.lower():
|
|
2938
|
+
score += 8
|
|
2939
|
+
reason = f"描述含'{kw}'"
|
|
2940
|
+
break
|
|
2941
|
+
|
|
2942
|
+
# 小尺寸可点击元素(可能是 X 按钮)
|
|
2943
|
+
if clickable and 30 < width < 200 and 30 < height < 200:
|
|
2944
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
2945
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
2946
|
+
|
|
2947
|
+
# 在屏幕右半边上半部分,很可能是 X
|
|
2948
|
+
if cx > screen_width * 0.6 and cy < screen_height * 0.5:
|
|
2949
|
+
score += 5
|
|
2950
|
+
reason = reason or "右上角小按钮"
|
|
2951
|
+
# 在屏幕上半部分的小按钮,也可能是 X
|
|
2952
|
+
elif cy < screen_height * 0.4:
|
|
2953
|
+
score += 2
|
|
2954
|
+
reason = reason or "上部小按钮"
|
|
2955
|
+
|
|
2956
|
+
# 只要是可点击的小按钮都考虑(即使没有文本)
|
|
2957
|
+
if score > 0 or (clickable and 30 < width < 150 and 30 < height < 150):
|
|
2958
|
+
if not reason and clickable:
|
|
2959
|
+
reason = "可点击小按钮"
|
|
2960
|
+
score = max(score, 1) # 确保有分数
|
|
2961
|
+
close_candidates.append({
|
|
2962
|
+
'score': score,
|
|
2963
|
+
'reason': reason,
|
|
2964
|
+
'bounds': (x1, y1, x2, y2),
|
|
2965
|
+
'center': (cx, cy),
|
|
2966
|
+
'resource_id': resource_id,
|
|
2967
|
+
'text': text
|
|
2968
|
+
})
|
|
2969
|
+
|
|
2970
|
+
# 按分数排序
|
|
2971
|
+
close_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
2972
|
+
|
|
2973
|
+
if close_candidates:
|
|
2974
|
+
best = close_candidates[0]
|
|
2975
|
+
cx, cy = best['center']
|
|
2976
|
+
bounds = best['bounds']
|
|
2977
|
+
|
|
2978
|
+
# 点击前截图(用于自动学习)
|
|
2979
|
+
pre_screenshot = None
|
|
2980
|
+
if auto_learn:
|
|
2981
|
+
pre_result = self.take_screenshot(description="关闭前", compress=False)
|
|
2982
|
+
pre_screenshot = pre_result.get("screenshot_path")
|
|
2983
|
+
|
|
2984
|
+
# 点击
|
|
2985
|
+
self.click_at_coords(cx, cy)
|
|
2986
|
+
time.sleep(0.5)
|
|
2987
|
+
|
|
2988
|
+
result["success"] = True
|
|
2989
|
+
result["method"] = "控件树"
|
|
2990
|
+
result["message"] = f"✅ 通过控件树找到关闭按钮并点击\n" \
|
|
2991
|
+
f" 位置: ({cx}, {cy})\n" \
|
|
2992
|
+
f" 原因: {best['reason']}"
|
|
2993
|
+
|
|
2994
|
+
# 自动学习:检查这个 X 是否已在模板库,不在就添加
|
|
2995
|
+
if auto_learn and pre_screenshot:
|
|
2996
|
+
learn_result = self._auto_learn_template(pre_screenshot, bounds)
|
|
2997
|
+
if learn_result:
|
|
2998
|
+
result["learned_template"] = learn_result
|
|
2999
|
+
result["message"] += f"\n📚 自动学习: {learn_result}"
|
|
3000
|
+
|
|
3001
|
+
return result
|
|
3002
|
+
|
|
3003
|
+
# ========== 第2步:模板匹配 ==========
|
|
3004
|
+
screenshot_path = None
|
|
3005
|
+
try:
|
|
3006
|
+
from .template_matcher import TemplateMatcher
|
|
3007
|
+
|
|
3008
|
+
# 截图用于模板匹配
|
|
3009
|
+
screenshot_result = self.take_screenshot(description="模板匹配", compress=False)
|
|
3010
|
+
screenshot_path = screenshot_result.get("screenshot_path")
|
|
3011
|
+
|
|
3012
|
+
if screenshot_path:
|
|
3013
|
+
matcher = TemplateMatcher()
|
|
3014
|
+
match_result = matcher.find_close_buttons(screenshot_path, threshold=0.75)
|
|
3015
|
+
|
|
3016
|
+
# 直接使用最佳匹配(已按置信度排序)
|
|
3017
|
+
if match_result.get("success") and match_result.get("best_match"):
|
|
3018
|
+
best = match_result["best_match"]
|
|
3019
|
+
x_pct = best["percent"]["x"]
|
|
3020
|
+
y_pct = best["percent"]["y"]
|
|
3021
|
+
|
|
3022
|
+
# 点击
|
|
3023
|
+
self.click_by_percent(x_pct, y_pct)
|
|
3024
|
+
time.sleep(0.5)
|
|
3025
|
+
|
|
3026
|
+
result["success"] = True
|
|
3027
|
+
result["method"] = "模板匹配"
|
|
3028
|
+
result["message"] = f"✅ 通过模板匹配找到关闭按钮并点击\n" \
|
|
3029
|
+
f" 模板: {best.get('template', 'unknown')}\n" \
|
|
3030
|
+
f" 置信度: {best.get('confidence', 'N/A')}%\n" \
|
|
3031
|
+
f" 位置: ({x_pct:.1f}%, {y_pct:.1f}%)"
|
|
3032
|
+
return result
|
|
3033
|
+
|
|
3034
|
+
except ImportError:
|
|
3035
|
+
pass # OpenCV 未安装,跳过模板匹配
|
|
3036
|
+
except Exception:
|
|
3037
|
+
pass # 模板匹配失败,继续下一步
|
|
3038
|
+
|
|
3039
|
+
# ========== 第3步:返回截图供 AI 分析 ==========
|
|
3040
|
+
if not screenshot_path:
|
|
3041
|
+
screenshot_result = self.take_screenshot(description="需要AI分析", compress=True)
|
|
3042
|
+
|
|
3043
|
+
result["success"] = False
|
|
3044
|
+
result["method"] = None
|
|
3045
|
+
result["message"] = "❌ 控件树和模板匹配都未找到关闭按钮\n" \
|
|
3046
|
+
"📸 已截图,请 AI 分析图片中的 X 按钮位置\n" \
|
|
3047
|
+
"💡 找到后使用 mobile_click_by_percent(x%, y%) 点击"
|
|
3048
|
+
result["screenshot"] = screenshot_result if not screenshot_path else {"screenshot_path": screenshot_path}
|
|
3049
|
+
result["need_ai_analysis"] = True
|
|
3050
|
+
|
|
3051
|
+
return result
|
|
3052
|
+
|
|
3053
|
+
except Exception as e:
|
|
3054
|
+
return {"success": False, "error": f"关闭弹窗失败: {e}"}
|
|
3055
|
+
|
|
3056
|
+
def _detect_popup_region(self, root) -> tuple:
|
|
3057
|
+
"""从控件树中检测弹窗区域
|
|
3058
|
+
|
|
3059
|
+
Args:
|
|
3060
|
+
root: 控件树根元素
|
|
3061
|
+
|
|
3062
|
+
Returns:
|
|
3063
|
+
弹窗边界 (x1, y1, x2, y2) 或 None
|
|
3064
|
+
"""
|
|
3065
|
+
import re
|
|
3066
|
+
|
|
3067
|
+
screen_width = self.client.u2.info.get('displayWidth', 1440)
|
|
3068
|
+
screen_height = self.client.u2.info.get('displayHeight', 3200)
|
|
3069
|
+
|
|
3070
|
+
popup_candidates = []
|
|
3071
|
+
|
|
3072
|
+
for elem in root.iter():
|
|
3073
|
+
bounds_str = elem.attrib.get('bounds', '')
|
|
3074
|
+
if not bounds_str:
|
|
3075
|
+
continue
|
|
3076
|
+
|
|
3077
|
+
match = re.match(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds_str)
|
|
3078
|
+
if not match:
|
|
3079
|
+
continue
|
|
3080
|
+
|
|
3081
|
+
x1, y1, x2, y2 = map(int, match.groups())
|
|
3082
|
+
width = x2 - x1
|
|
3083
|
+
height = y2 - y1
|
|
3084
|
+
|
|
3085
|
+
# 弹窗特征:
|
|
3086
|
+
# 1. 不是全屏
|
|
3087
|
+
# 2. 在屏幕中央
|
|
3088
|
+
# 3. 有一定大小
|
|
3089
|
+
is_fullscreen = (width >= screen_width * 0.95 and height >= screen_height * 0.9)
|
|
3090
|
+
is_centered = (x1 > screen_width * 0.05 and x2 < screen_width * 0.95)
|
|
3091
|
+
is_reasonable_size = (width > 200 and height > 200 and
|
|
3092
|
+
width < screen_width * 0.95 and
|
|
3093
|
+
height < screen_height * 0.8)
|
|
3094
|
+
|
|
3095
|
+
if not is_fullscreen and is_centered and is_reasonable_size:
|
|
3096
|
+
# 计算"弹窗感"分数
|
|
3097
|
+
area = width * height
|
|
3098
|
+
center_x = (x1 + x2) / 2
|
|
3099
|
+
center_y = (y1 + y2) / 2
|
|
3100
|
+
center_dist = abs(center_x - screen_width/2) + abs(center_y - screen_height/2)
|
|
3101
|
+
|
|
3102
|
+
score = area / 1000 - center_dist / 10
|
|
3103
|
+
popup_candidates.append({
|
|
3104
|
+
'bounds': (x1, y1, x2, y2),
|
|
3105
|
+
'score': score
|
|
3106
|
+
})
|
|
3107
|
+
|
|
3108
|
+
if popup_candidates:
|
|
3109
|
+
# 返回分数最高的弹窗
|
|
3110
|
+
popup_candidates.sort(key=lambda x: x['score'], reverse=True)
|
|
3111
|
+
return popup_candidates[0]['bounds']
|
|
3112
|
+
|
|
3113
|
+
return None
|
|
3114
|
+
|
|
3115
|
+
def _auto_learn_template(self, screenshot_path: str, bounds: tuple, threshold: float = 0.6) -> str:
|
|
3116
|
+
"""自动学习:检查 X 按钮是否已在模板库,不在就添加
|
|
3117
|
+
|
|
3118
|
+
Args:
|
|
3119
|
+
screenshot_path: 截图路径
|
|
3120
|
+
bounds: X 按钮的边界 (x1, y1, x2, y2)
|
|
3121
|
+
threshold: 判断是否已存在的阈值(高于此值认为已存在)
|
|
3122
|
+
|
|
3123
|
+
Returns:
|
|
3124
|
+
新模板名称,如果是新模板的话;已存在或失败返回 None
|
|
3125
|
+
"""
|
|
3126
|
+
try:
|
|
3127
|
+
from .template_matcher import TemplateMatcher
|
|
3128
|
+
from PIL import Image
|
|
3129
|
+
import time
|
|
3130
|
+
|
|
3131
|
+
x1, y1, x2, y2 = bounds
|
|
3132
|
+
cx, cy = (x1 + x2) // 2, (y1 + y2) // 2
|
|
3133
|
+
width = x2 - x1
|
|
3134
|
+
height = y2 - y1
|
|
3135
|
+
|
|
3136
|
+
# 扩展一点边界,确保裁剪完整
|
|
3137
|
+
padding = max(10, int(max(width, height) * 0.2))
|
|
3138
|
+
|
|
3139
|
+
# 打开截图
|
|
3140
|
+
img = Image.open(screenshot_path)
|
|
3141
|
+
|
|
3142
|
+
# 裁剪 X 按钮区域
|
|
3143
|
+
crop_x1 = max(0, x1 - padding)
|
|
3144
|
+
crop_y1 = max(0, y1 - padding)
|
|
3145
|
+
crop_x2 = min(img.width, x2 + padding)
|
|
3146
|
+
crop_y2 = min(img.height, y2 + padding)
|
|
3147
|
+
|
|
3148
|
+
cropped = img.crop((crop_x1, crop_y1, crop_x2, crop_y2))
|
|
3149
|
+
|
|
3150
|
+
# 保存临时文件用于匹配检查
|
|
3151
|
+
temp_path = self.screenshot_dir / "temp_new_x.png"
|
|
3152
|
+
cropped.save(str(temp_path))
|
|
3153
|
+
|
|
3154
|
+
# 检查是否已在模板库中(用模板匹配检测相似度)
|
|
3155
|
+
matcher = TemplateMatcher()
|
|
3156
|
+
|
|
3157
|
+
import cv2
|
|
3158
|
+
new_img = cv2.imread(str(temp_path), cv2.IMREAD_GRAYSCALE)
|
|
3159
|
+
if new_img is None:
|
|
3160
|
+
return None
|
|
3161
|
+
|
|
3162
|
+
is_new = True
|
|
3163
|
+
for template_file in matcher.template_dir.glob("*.png"):
|
|
3164
|
+
template = cv2.imread(str(template_file), cv2.IMREAD_GRAYSCALE)
|
|
3165
|
+
if template is None:
|
|
3166
|
+
continue
|
|
3167
|
+
|
|
3168
|
+
# 将两个图都调整到合适大小,然后用小模板在大图中搜索
|
|
3169
|
+
# 这样比较更接近实际匹配场景
|
|
3170
|
+
|
|
3171
|
+
# 新图作为搜索区域(稍大一点)
|
|
3172
|
+
new_resized = cv2.resize(new_img, (100, 100))
|
|
3173
|
+
# 模板调整到较小尺寸
|
|
3174
|
+
template_resized = cv2.resize(template, (60, 60))
|
|
3175
|
+
|
|
3176
|
+
# 在新图中搜索模板
|
|
3177
|
+
result = cv2.matchTemplate(new_resized, template_resized, cv2.TM_CCOEFF_NORMED)
|
|
3178
|
+
_, max_val, _, _ = cv2.minMaxLoc(result)
|
|
3179
|
+
|
|
3180
|
+
if max_val >= threshold:
|
|
3181
|
+
is_new = False
|
|
3182
|
+
break
|
|
3183
|
+
|
|
3184
|
+
# 清理临时文件
|
|
3185
|
+
if temp_path.exists():
|
|
3186
|
+
temp_path.unlink()
|
|
3187
|
+
|
|
3188
|
+
if is_new:
|
|
3189
|
+
# 生成唯一模板名
|
|
3190
|
+
timestamp = time.strftime("%m%d_%H%M%S")
|
|
3191
|
+
template_name = f"auto_x_{timestamp}.png"
|
|
3192
|
+
template_path = matcher.template_dir / template_name
|
|
3193
|
+
|
|
3194
|
+
# 保存新模板
|
|
3195
|
+
cropped.save(str(template_path))
|
|
3196
|
+
|
|
3197
|
+
return template_name
|
|
3198
|
+
else:
|
|
3199
|
+
return None # 已存在类似模板
|
|
3200
|
+
|
|
3201
|
+
except Exception as e:
|
|
3202
|
+
return None # 学习失败,不影响主流程
|
|
3203
|
+
|
|
3204
|
+
def template_add_by_percent(self, x_percent: float, y_percent: float,
|
|
3205
|
+
size: int, template_name: str) -> Dict:
|
|
3206
|
+
"""通过百分比坐标添加模板(更方便!)
|
|
3207
|
+
|
|
3208
|
+
自动截图 → 根据百分比位置裁剪 → 保存为模板
|
|
3209
|
+
|
|
3210
|
+
Args:
|
|
3211
|
+
x_percent: X号中心的水平百分比 (0-100)
|
|
3212
|
+
y_percent: X号中心的垂直百分比 (0-100)
|
|
3213
|
+
size: 裁剪区域大小(正方形边长,像素)
|
|
3214
|
+
template_name: 模板名称
|
|
3215
|
+
|
|
3216
|
+
Returns:
|
|
3217
|
+
结果
|
|
3218
|
+
"""
|
|
3219
|
+
try:
|
|
3220
|
+
from .template_matcher import TemplateMatcher
|
|
3221
|
+
from PIL import Image
|
|
3222
|
+
|
|
3223
|
+
# 先截图(不带 SoM 标注的干净截图)
|
|
3224
|
+
screenshot_result = self.take_screenshot(description="添加模板", compress=False)
|
|
3225
|
+
screenshot_path = screenshot_result.get("screenshot_path")
|
|
3226
|
+
|
|
3227
|
+
if not screenshot_path:
|
|
3228
|
+
return {"success": False, "error": "截图失败"}
|
|
3229
|
+
|
|
3230
|
+
# 读取截图获取尺寸
|
|
3231
|
+
img = Image.open(screenshot_path)
|
|
3232
|
+
img_w, img_h = img.size
|
|
3233
|
+
|
|
3234
|
+
# 计算中心点像素坐标
|
|
3235
|
+
cx = int(img_w * x_percent / 100)
|
|
3236
|
+
cy = int(img_h * y_percent / 100)
|
|
3237
|
+
|
|
3238
|
+
# 计算裁剪区域
|
|
3239
|
+
half = size // 2
|
|
3240
|
+
x1 = max(0, cx - half)
|
|
3241
|
+
y1 = max(0, cy - half)
|
|
3242
|
+
x2 = min(img_w, cx + half)
|
|
3243
|
+
y2 = min(img_h, cy + half)
|
|
3244
|
+
|
|
3245
|
+
# 裁剪并保存
|
|
3246
|
+
cropped = img.crop((x1, y1, x2, y2))
|
|
3247
|
+
|
|
3248
|
+
matcher = TemplateMatcher()
|
|
3249
|
+
output_path = matcher.template_dir / f"{template_name}.png"
|
|
3250
|
+
cropped.save(str(output_path))
|
|
3251
|
+
|
|
3252
|
+
return {
|
|
3253
|
+
"success": True,
|
|
3254
|
+
"message": f"✅ 模板已保存: {template_name}",
|
|
3255
|
+
"template_path": str(output_path),
|
|
3256
|
+
"center_percent": f"({x_percent}%, {y_percent}%)",
|
|
3257
|
+
"center_pixel": f"({cx}, {cy})",
|
|
3258
|
+
"crop_region": f"({x1},{y1}) - ({x2},{y2})",
|
|
3259
|
+
"size": f"{cropped.size[0]}x{cropped.size[1]}"
|
|
3260
|
+
}
|
|
3261
|
+
|
|
3262
|
+
except ImportError as e:
|
|
3263
|
+
return {"success": False, "error": f"需要安装依赖: {e}"}
|
|
3264
|
+
except Exception as e:
|
|
3265
|
+
return {"success": False, "error": f"添加模板失败: {e}"}
|
|
3266
|
+
|