mobile-mcp-ai 2.1.2__py3-none-any.whl → 2.5.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. mobile_mcp/__init__.py +34 -0
  2. mobile_mcp/config.py +142 -0
  3. mobile_mcp/core/basic_tools_lite.py +3266 -0
  4. {core → mobile_mcp/core}/device_manager.py +2 -2
  5. mobile_mcp/core/dynamic_config.py +272 -0
  6. mobile_mcp/core/ios_client_wda.py +569 -0
  7. mobile_mcp/core/ios_device_manager_wda.py +306 -0
  8. {core → mobile_mcp/core}/mobile_client.py +279 -39
  9. mobile_mcp/core/template_matcher.py +429 -0
  10. mobile_mcp/core/templates/close_buttons/auto_x_0112_151217.png +0 -0
  11. mobile_mcp/core/templates/close_buttons/auto_x_0112_152037.png +0 -0
  12. mobile_mcp/core/templates/close_buttons/auto_x_0112_152840.png +0 -0
  13. mobile_mcp/core/templates/close_buttons/auto_x_0112_153256.png +0 -0
  14. mobile_mcp/core/templates/close_buttons/auto_x_0112_154847.png +0 -0
  15. mobile_mcp/core/templates/close_buttons/gray_x_stock_ad.png +0 -0
  16. {core → mobile_mcp/core}/utils/smart_wait.py +3 -3
  17. mobile_mcp/mcp_tools/__init__.py +10 -0
  18. mobile_mcp/mcp_tools/mcp_server.py +1071 -0
  19. mobile_mcp_ai-2.5.8.dist-info/METADATA +469 -0
  20. mobile_mcp_ai-2.5.8.dist-info/RECORD +32 -0
  21. mobile_mcp_ai-2.5.8.dist-info/entry_points.txt +2 -0
  22. mobile_mcp_ai-2.5.8.dist-info/licenses/LICENSE +201 -0
  23. mobile_mcp_ai-2.5.8.dist-info/top_level.txt +1 -0
  24. core/ai/__init__.py +0 -11
  25. core/ai/ai_analyzer.py +0 -197
  26. core/ai/ai_config.py +0 -116
  27. core/ai/ai_platform_adapter.py +0 -399
  28. core/ai/smart_test_executor.py +0 -520
  29. core/ai/test_generator.py +0 -365
  30. core/ai/test_generator_from_history.py +0 -391
  31. core/ai/test_generator_standalone.py +0 -293
  32. core/assertion/__init__.py +0 -9
  33. core/assertion/smart_assertion.py +0 -341
  34. core/basic_tools.py +0 -377
  35. core/h5/__init__.py +0 -10
  36. core/h5/h5_handler.py +0 -548
  37. core/ios_client.py +0 -219
  38. core/ios_device_manager.py +0 -252
  39. core/locator/__init__.py +0 -10
  40. core/locator/cursor_ai_auto_analyzer.py +0 -119
  41. core/locator/cursor_vision_helper.py +0 -414
  42. core/locator/mobile_smart_locator.py +0 -1640
  43. core/locator/position_analyzer.py +0 -813
  44. core/locator/script_updater.py +0 -157
  45. core/nl_test_runner.py +0 -585
  46. core/smart_app_launcher.py +0 -334
  47. core/smart_tools.py +0 -311
  48. mcp/__init__.py +0 -8
  49. mcp/mcp_server.py +0 -1919
  50. mcp/mcp_server_simple.py +0 -476
  51. mobile_mcp_ai-2.1.2.dist-info/METADATA +0 -567
  52. mobile_mcp_ai-2.1.2.dist-info/RECORD +0 -45
  53. mobile_mcp_ai-2.1.2.dist-info/entry_points.txt +0 -2
  54. mobile_mcp_ai-2.1.2.dist-info/top_level.txt +0 -4
  55. vision/__init__.py +0 -10
  56. vision/vision_locator.py +0 -404
  57. {core → mobile_mcp/core}/__init__.py +0 -0
  58. {core → mobile_mcp/core}/utils/__init__.py +0 -0
  59. {core → mobile_mcp/core}/utils/logger.py +0 -0
  60. {core → mobile_mcp/core}/utils/operation_history_manager.py +0 -0
  61. {utils → mobile_mcp/utils}/__init__.py +0 -0
  62. {utils → mobile_mcp/utils}/logger.py +0 -0
  63. {utils → mobile_mcp/utils}/xml_formatter.py +0 -0
  64. {utils → mobile_mcp/utils}/xml_parser.py +0 -0
  65. {mobile_mcp_ai-2.1.2.dist-info → mobile_mcp_ai-2.5.8.dist-info}/WHEEL +0 -0
@@ -1,813 +0,0 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- """
4
- 位置分析器 - 通过XML的bounds信息定位无标识元素
5
-
6
- 核心思路:
7
- 1. 底部导航栏图标虽然没有text/desc/id,但有bounds坐标
8
- 2. 通过分析bounds的位置(Y坐标、X坐标)来定位
9
- 3. 完全免费,速度快(50-100ms)
10
-
11
- 适用场景:
12
- ✓ 底部导航栏图标(Y坐标在底部,X坐标均匀分布)
13
- ✓ 顶部导航栏图标(Y坐标在顶部)
14
- ✓ 悬浮按钮(固定位置)
15
- ✓ 网格布局的图标(如九宫格)
16
- """
17
- import sys
18
- import re
19
- from typing import List, Dict, Optional, Tuple
20
-
21
-
22
- class PositionAnalyzer:
23
- """位置分析器"""
24
-
25
- def __init__(self, screen_width: int = 1080, screen_height: int = 2400):
26
- """
27
- 初始化位置分析器
28
-
29
- Args:
30
- screen_width: 屏幕宽度(默认1080)
31
- screen_height: 屏幕高度(默认2400)
32
- """
33
- self.screen_width = screen_width
34
- self.screen_height = screen_height
35
-
36
- # 定义区域(可根据实际屏幕调整)
37
- self.regions = {
38
- 'top': (0, int(screen_height * 0.1)), # 顶部区域:0-10%
39
- 'bottom': (int(screen_height * 0.85), screen_height), # 底部区域:85-100%
40
- 'left': (0, int(screen_width * 0.2)), # 左侧区域:0-20%
41
- 'right': (int(screen_width * 0.8), screen_width), # 右侧区域:80-100%
42
- }
43
-
44
- def analyze_nth_element(self, elements: List[Dict], query: str) -> Optional[Dict]:
45
- """
46
- 分析"第N个"元素(通用方法)
47
-
48
- 支持的描述:
49
- - "第一个帖子"、"第二个帖子"、"第三个帖子"
50
- - "第1个按钮"、"第2个图标"
51
- - "第一个可点击元素"
52
-
53
- Args:
54
- elements: 所有元素列表
55
- query: 查询文本
56
-
57
- Returns:
58
- 匹配的元素信息
59
- """
60
- # 提取序号
61
- index = self._extract_index(query)
62
- if index is None:
63
- return None
64
-
65
- print(f" 📍 位置分析:第{index}个元素", file=sys.stderr)
66
-
67
- # 提取关键词(帖子、按钮、图标等)
68
- keywords = []
69
- if '帖子' in query or '帖' in query:
70
- keywords = ['帖子', '帖']
71
- elif '按钮' in query:
72
- keywords = ['按钮', 'button']
73
- elif '图标' in query:
74
- keywords = ['图标', 'icon', 'image']
75
- elif '文本' in query or '文字' in query:
76
- keywords = ['文本', 'text']
77
-
78
- # 1. 筛选候选元素
79
- candidates = []
80
-
81
- # 如果有关键词,先按关键词筛选
82
- if keywords:
83
- for elem in elements:
84
- # 跳过系统栏元素
85
- if self._is_system_ui(elem):
86
- continue
87
-
88
- # 检查class_name是否包含关键词
89
- class_name = elem.get('class_name', '').lower()
90
- text = elem.get('text', '').lower()
91
- desc = elem.get('content_desc', '').lower()
92
-
93
- # 帖子通常是可点击的、有一定大小的容器
94
- if '帖' in keywords:
95
- if elem.get('clickable', False) or elem.get('long_clickable', False):
96
- bounds = self._get_bounds(elem)
97
- if bounds:
98
- x1, y1, x2, y2 = bounds
99
- width = x2 - x1
100
- height = y2 - y1
101
- center_y = (y1 + y2) // 2
102
- center_x = (x1 + x2) // 2
103
-
104
- # 帖子卡片特征:
105
- # 1. 宽度较大(至少屏幕宽度的50%,优先选择接近屏幕宽度的)
106
- # 2. 高度在150-800px之间
107
- # 3. 位于屏幕中间区域(Y坐标在200-2000之间,避开状态栏和底部导航栏)
108
- # 4. 不是异常小的元素(避免选择帖子内部的图标、按钮等)
109
- # 5. 过滤掉小的ImageView(通常是标签、图标,不是帖子卡片)
110
- is_reasonable_width = (self.screen_width * 0.5 <= width <= self.screen_width * 1.1)
111
- is_reasonable_height = (150 <= height <= 800)
112
- is_middle_area = (200 <= center_y <= 2000)
113
- is_not_too_small = (width * height > 50000) # 面积至少50000像素
114
- is_not_small_imageview = not (class_name.lower() == 'imageview' and width < 400 and height < 300)
115
-
116
- if is_reasonable_width and is_reasonable_height and is_middle_area and is_not_too_small and is_not_small_imageview:
117
- candidates.append(elem)
118
- else:
119
- # 调试信息(只在详细模式下打印)
120
- pass
121
- # 按钮
122
- elif '按钮' in keywords or 'button' in keywords:
123
- if elem.get('clickable', False) or 'button' in class_name:
124
- candidates.append(elem)
125
- # 图标
126
- elif '图标' in keywords or 'icon' in keywords or 'image' in keywords:
127
- if 'image' in class_name or elem.get('clickable', False):
128
- candidates.append(elem)
129
- # 文本
130
- elif '文本' in keywords or 'text' in keywords:
131
- if 'text' in class_name and (text or desc):
132
- candidates.append(elem)
133
- else:
134
- # 没有关键词,默认选择所有可点击元素
135
- for elem in elements:
136
- if self._is_system_ui(elem):
137
- continue
138
- if elem.get('clickable', False) or elem.get('long_clickable', False):
139
- candidates.append(elem)
140
-
141
- print(f" → 找到 {len(candidates)} 个候选元素", file=sys.stderr)
142
-
143
- if not candidates:
144
- return None
145
-
146
- # 2. 按Y坐标(从上到下)排序
147
- sorted_candidates = sorted(candidates, key=lambda e: self._get_center_y(e))
148
-
149
- # 3. 选择第N个
150
- if index > len(sorted_candidates):
151
- print(f" ❌ 只有 {len(sorted_candidates)} 个元素,无法选择第 {index} 个", file=sys.stderr)
152
- return None
153
-
154
- selected = sorted_candidates[index - 1] # 转换为0-based索引
155
- center_x, center_y = self._get_center(selected)
156
- bounds = selected.get('bounds', '')
157
-
158
- print(f" ✅ 选择第{index}个元素:", file=sys.stderr)
159
- print(f" class: {selected.get('class_name', 'Unknown')}", file=sys.stderr)
160
- print(f" text: {selected.get('text', '')}", file=sys.stderr)
161
- print(f" desc: {selected.get('content_desc', '')}", file=sys.stderr)
162
- print(f" 中心点: ({center_x}, {center_y})", file=sys.stderr)
163
- print(f" bounds: {bounds}", file=sys.stderr)
164
-
165
- # 返回结果
166
- return {
167
- 'element': query,
168
- 'ref': bounds, # 使用bounds作为ref
169
- 'confidence': 90,
170
- 'method': 'position_analysis_nth',
171
- 'x': center_x,
172
- 'y': center_y,
173
- }
174
-
175
- def analyze_floating_button(self, elements: List[Dict], query: str) -> Optional[Dict]:
176
- """
177
- 分析悬浮按钮(FloatingActionButton)
178
-
179
- 特征:
180
- - 通常在右下角或底部中间
181
- - 大小接近正方形(100-300px)
182
- - Y坐标在1700-2100之间
183
- - 可点击
184
- - 通常没有text/desc
185
-
186
- Args:
187
- elements: 所有元素列表
188
- query: 查询文本(如"最下面悬浮按钮"、"右下角加号")
189
-
190
- Returns:
191
- 匹配的元素信息
192
- """
193
- print(f" 📍 位置分析:悬浮按钮", file=sys.stderr)
194
-
195
- # 1. 筛选候选元素
196
- candidates = []
197
- for elem in elements:
198
- if not elem.get('clickable', False):
199
- continue
200
-
201
- bounds = self._get_bounds(elem)
202
- if not bounds:
203
- continue
204
-
205
- x1, y1, x2, y2 = bounds
206
- width = x2 - x1
207
- height = y2 - y1
208
- center_x = (x1 + x2) // 2
209
- center_y = (y1 + y2) // 2
210
-
211
- # 悬浮按钮特征:
212
- # 1. Y坐标在1700-2100之间(底部但不是最底部)
213
- # 2. 大小在100-300之间
214
- # 3. 接近正方形(宽高比0.7-1.3)
215
- if 1700 < center_y < 2100:
216
- if 100 < width < 300 and 100 < height < 300:
217
- ratio = width / height if height > 0 else 0
218
- if 0.7 < ratio < 1.3:
219
- candidates.append({
220
- 'elem': elem,
221
- 'center': (center_x, center_y),
222
- 'size': (width, height),
223
- 'bounds': elem.get('bounds', ''),
224
- })
225
-
226
- print(f" → 找到 {len(candidates)} 个悬浮按钮候选", file=sys.stderr)
227
-
228
- if not candidates:
229
- return None
230
-
231
- # 2. 打印候选元素
232
- print(f" 📋 悬浮按钮候选元素:", file=sys.stderr)
233
- for i, cand in enumerate(candidates, 1):
234
- print(f" [{i}] 中心点{cand['center']}, 大小{cand['size']}, bounds={cand['bounds']}", file=sys.stderr)
235
-
236
- # 3. 根据查询选择
237
- if "最下面" in query or "最下方" in query:
238
- # 选择Y坐标最大的(最下面的)
239
- selected = max(candidates, key=lambda c: c['center'][1])
240
- print(f" ✅ 选择最下面的悬浮按钮: 中心点{selected['center']}", file=sys.stderr)
241
- elif "右下角" in query or "右下" in query:
242
- # 选择右下角的(X最大,Y最大)
243
- selected = max(candidates, key=lambda c: (c['center'][0] + c['center'][1]))
244
- print(f" ✅ 选择右下角的悬浮按钮: 中心点{selected['center']}", file=sys.stderr)
245
- else:
246
- # 默认选择最下面的
247
- selected = max(candidates, key=lambda c: c['center'][1])
248
- print(f" ✅ 默认选择最下面的悬浮按钮: 中心点{selected['center']}", file=sys.stderr)
249
-
250
- return {
251
- 'element': query,
252
- 'ref': selected['bounds'],
253
- 'confidence': 95,
254
- 'method': 'position_analysis_fab',
255
- 'x': selected['center'][0],
256
- 'y': selected['center'][1],
257
- }
258
-
259
- def analyze_bottom_navigation(self, elements: List[Dict], query: str) -> Optional[Dict]:
260
- """
261
- 分析底部导航栏
262
-
263
- Args:
264
- elements: 所有元素列表
265
- query: 查询文本(如"底部导航栏第3个图标")
266
-
267
- Returns:
268
- 匹配的元素信息
269
- """
270
- print(f" 📍 位置分析:底部导航栏", file=sys.stderr)
271
-
272
- # 1. 筛选底部区域的元素
273
- bottom_elements = self._filter_by_region(elements, 'bottom')
274
- print(f" → 底部区域元素: {len(bottom_elements)}个", file=sys.stderr)
275
-
276
- # 2. 筛选可点击的元素(导航栏图标通常是clickable)
277
- clickable_bottom = [e for e in bottom_elements if e.get('clickable', False)]
278
- print(f" → 可点击元素: {len(clickable_bottom)}个", file=sys.stderr)
279
-
280
- if not clickable_bottom:
281
- print(f" ❌ 底部没有可点击元素", file=sys.stderr)
282
- return None
283
-
284
- # 2.5. 过滤掉异常宽的元素(如全屏宽度的View)
285
- # 导航栏图标通常宽度在 50-300 之间
286
- filtered_elements = []
287
- for elem in clickable_bottom:
288
- bounds = self._get_bounds(elem)
289
- if bounds:
290
- x1, y1, x2, y2 = bounds
291
- width = x2 - x1
292
- # 过滤掉宽度 > 500 或 < 50 的元素
293
- if 50 <= width <= 500:
294
- filtered_elements.append(elem)
295
-
296
- if filtered_elements:
297
- print(f" → 过滤后元素: {len(filtered_elements)}个(过滤掉{len(clickable_bottom) - len(filtered_elements)}个异常宽度元素)", file=sys.stderr)
298
- clickable_bottom = filtered_elements
299
-
300
- # 3. 按X坐标排序(从左到右)
301
- sorted_elements = sorted(clickable_bottom, key=lambda e: self._get_center_x(e))
302
-
303
- # 4. 打印所有候选元素
304
- print(f" 📋 底部导航栏候选元素(从左到右):", file=sys.stderr)
305
- for i, elem in enumerate(sorted_elements, 1):
306
- bounds = elem.get('bounds', '')
307
- center_x, center_y = self._get_center(elem)
308
- class_name = elem.get('class_name', '')
309
- text = elem.get('text', '')
310
- desc = elem.get('content_desc', '')
311
-
312
- info = f"class={class_name}"
313
- if text:
314
- info += f", text='{text}'"
315
- if desc:
316
- info += f", desc='{desc[:20]}'"
317
-
318
- print(f" [{i}] 中心点({center_x}, {center_y}) | bounds={bounds} | {info}", file=sys.stderr)
319
-
320
- # 5. 根据查询提取索引
321
- index = self._extract_index(query)
322
-
323
- if index is None:
324
- # 没有明确索引,尝试关键词匹配
325
- print(f" ⚠️ 查询中没有明确索引,尝试关键词匹配...", file=sys.stderr)
326
- return self._match_by_keyword(sorted_elements, query)
327
-
328
- if index < 1 or index > len(sorted_elements):
329
- print(f" ❌ 索引超出范围: {index}(共{len(sorted_elements)}个元素)", file=sys.stderr)
330
- return None
331
-
332
- # 6. 返回对应索引的元素
333
- selected = sorted_elements[index - 1]
334
- bounds = selected.get('bounds', '')
335
- center_x, center_y = self._get_center(selected)
336
-
337
- print(f" ✅ 选择第{index}个元素:", file=sys.stderr)
338
- print(f" 中心点: ({center_x}, {center_y})", file=sys.stderr)
339
- print(f" bounds: {bounds}", file=sys.stderr)
340
-
341
- return {
342
- 'element': query,
343
- 'ref': bounds, # 使用bounds作为ref
344
- 'confidence': 95,
345
- 'method': 'position_analysis',
346
- 'x': center_x,
347
- 'y': center_y,
348
- }
349
-
350
- def analyze_corner_position(self, elements: List[Dict], query: str, corner: str = 'top_right') -> Optional[Dict]:
351
- """
352
- 分析角落位置(右上角、左上角、右下角、左下角)
353
-
354
- Args:
355
- elements: 所有元素列表
356
- query: 查询文本(如"右上角搜索图标")
357
- corner: 角落位置('top_right', 'top_left', 'bottom_right', 'bottom_left')
358
-
359
- Returns:
360
- 匹配的元素信息
361
- """
362
- print(f" 📍 位置分析:{corner}角落", file=sys.stderr)
363
-
364
- # 定义角落区域(屏幕的20%区域,从10%扩大到20%以覆盖更多实际场景)
365
- corner_threshold = 0.2 # 20%
366
-
367
- # 根据角落类型定义筛选条件
368
- if corner == 'top_right':
369
- # 右上角:X坐标在右侧10%,Y坐标在顶部10%
370
- x_min = self.screen_width * (1 - corner_threshold)
371
- y_max = self.screen_height * corner_threshold
372
- elif corner == 'top_left':
373
- # 左上角:X坐标在左侧10%,Y坐标在顶部10%
374
- x_max = self.screen_width * corner_threshold
375
- y_max = self.screen_height * corner_threshold
376
- elif corner == 'bottom_right':
377
- # 右下角:X坐标在右侧10%,Y坐标在底部10%
378
- x_min = self.screen_width * (1 - corner_threshold)
379
- y_min = self.screen_height * (1 - corner_threshold)
380
- elif corner == 'bottom_left':
381
- # 左下角:X坐标在左侧10%,Y坐标在底部10%
382
- x_max = self.screen_width * corner_threshold
383
- y_min = self.screen_height * (1 - corner_threshold)
384
- else:
385
- return None
386
-
387
- # 1. 筛选候选元素(可点击的图标元素)
388
- candidates = []
389
- for elem in elements:
390
- if not elem.get('clickable', False):
391
- continue
392
-
393
- # 如果是图标查询,优先选择Image/ImageView类型
394
- if '图标' in query:
395
- class_name = elem.get('class_name', '').lower()
396
- if 'image' not in class_name and class_name not in ['imageview', 'imagebutton']:
397
- continue
398
-
399
- bounds = self._get_bounds(elem)
400
- if not bounds:
401
- continue
402
-
403
- x1, y1, x2, y2 = bounds
404
- center_x = (x1 + x2) // 2
405
- center_y = (y1 + y2) // 2
406
-
407
- # 检查是否在角落区域
408
- in_corner = False
409
- if corner == 'top_right':
410
- in_corner = center_x >= x_min and center_y <= y_max
411
- elif corner == 'top_left':
412
- in_corner = center_x <= x_max and center_y <= y_max
413
- elif corner == 'bottom_right':
414
- in_corner = center_x >= x_min and center_y >= y_min
415
- elif corner == 'bottom_left':
416
- in_corner = center_x <= x_max and center_y >= y_min
417
-
418
- if in_corner:
419
- candidates.append({
420
- 'elem': elem,
421
- 'center': (center_x, center_y),
422
- 'bounds': elem.get('bounds', ''),
423
- })
424
-
425
- print(f" → 找到 {len(candidates)} 个{corner}角落候选元素", file=sys.stderr)
426
-
427
- if not candidates:
428
- return None
429
-
430
- # 2. 如果有多个候选,选择最接近角落的(距离角落最近)
431
- if len(candidates) > 1:
432
- # 计算每个候选到角落的距离
433
- for cand in candidates:
434
- center_x, center_y = cand['center']
435
- if corner == 'top_right':
436
- # 距离右上角的距离(越小越好)
437
- distance = (self.screen_width - center_x) + center_y
438
- elif corner == 'top_left':
439
- distance = center_x + center_y
440
- elif corner == 'bottom_right':
441
- distance = (self.screen_width - center_x) + (self.screen_height - center_y)
442
- elif corner == 'bottom_left':
443
- distance = center_x + (self.screen_height - center_y)
444
- else:
445
- distance = 0
446
- cand['distance'] = distance
447
-
448
- # 选择距离最小的
449
- selected = min(candidates, key=lambda c: c['distance'])
450
- else:
451
- selected = candidates[0]
452
-
453
- center_x, center_y = selected['center']
454
- bounds = selected['bounds']
455
-
456
- print(f" ✅ 选择{corner}角落元素:", file=sys.stderr)
457
- print(f" 中心点: ({center_x}, {center_y})", file=sys.stderr)
458
- print(f" bounds: {bounds}", file=sys.stderr)
459
-
460
- return {
461
- 'element': query,
462
- 'ref': bounds,
463
- 'confidence': 95,
464
- 'method': 'position_analysis_corner',
465
- 'x': center_x,
466
- 'y': center_y,
467
- }
468
-
469
- def analyze_top_navigation(self, elements: List[Dict], query: str) -> Optional[Dict]:
470
- """
471
- 分析顶部导航栏
472
-
473
- Args:
474
- elements: 所有元素列表
475
- query: 查询文本(如"顶部第2个图标")
476
-
477
- Returns:
478
- 匹配的元素信息
479
- """
480
- print(f" 📍 位置分析:顶部导航栏", file=sys.stderr)
481
-
482
- # 1. 筛选顶部区域的元素
483
- top_elements = self._filter_by_region(elements, 'top')
484
- print(f" → 顶部区域元素: {len(top_elements)}个", file=sys.stderr)
485
-
486
- # 2. 筛选可点击的元素
487
- clickable_top = [e for e in top_elements if e.get('clickable', False)]
488
- print(f" → 可点击元素: {len(clickable_top)}个", file=sys.stderr)
489
-
490
- if not clickable_top:
491
- print(f" ❌ 顶部没有可点击元素", file=sys.stderr)
492
- return None
493
-
494
- # 3. 按X坐标排序(从左到右)
495
- sorted_elements = sorted(clickable_top, key=lambda e: self._get_center_x(e))
496
-
497
- # 4. 根据查询提取索引
498
- index = self._extract_index(query)
499
-
500
- if index is None or index < 1 or index > len(sorted_elements):
501
- print(f" ❌ 无法确定索引", file=sys.stderr)
502
- return None
503
-
504
- # 5. 返回对应索引的元素
505
- selected = sorted_elements[index - 1]
506
- bounds = selected.get('bounds', '')
507
- center_x, center_y = self._get_center(selected)
508
-
509
- print(f" ✅ 选择第{index}个元素: 中心点({center_x}, {center_y})", file=sys.stderr)
510
-
511
- return {
512
- 'element': query,
513
- 'ref': bounds,
514
- 'confidence': 95,
515
- 'method': 'position_analysis',
516
- 'x': center_x,
517
- 'y': center_y,
518
- }
519
-
520
- def analyze_grid_layout(self, elements: List[Dict], query: str, rows: int = 3, cols: int = 3) -> Optional[Dict]:
521
- """
522
- 分析网格布局(如九宫格)
523
-
524
- Args:
525
- elements: 所有元素列表
526
- query: 查询文本(如"第2行第3列的图标")
527
- rows: 行数
528
- cols: 列数
529
-
530
- Returns:
531
- 匹配的元素信息
532
- """
533
- print(f" 📍 位置分析:网格布局 ({rows}x{cols})", file=sys.stderr)
534
-
535
- # 1. 筛选可点击的元素
536
- clickable = [e for e in elements if e.get('clickable', False)]
537
-
538
- # 2. 按Y坐标分组(行)
539
- rows_groups = self._group_by_y(clickable, rows)
540
-
541
- # 3. 每行按X坐标排序(列)
542
- grid = []
543
- for row in rows_groups:
544
- sorted_row = sorted(row, key=lambda e: self._get_center_x(e))
545
- grid.append(sorted_row)
546
-
547
- # 4. 提取行列索引
548
- row_idx, col_idx = self._extract_grid_index(query)
549
-
550
- if row_idx is None or col_idx is None:
551
- print(f" ❌ 无法解析网格索引", file=sys.stderr)
552
- return None
553
-
554
- if row_idx >= len(grid) or col_idx >= len(grid[row_idx]):
555
- print(f" ❌ 索引超出范围", file=sys.stderr)
556
- return None
557
-
558
- # 5. 返回对应位置的元素
559
- selected = grid[row_idx][col_idx]
560
- bounds = selected.get('bounds', '')
561
- center_x, center_y = self._get_center(selected)
562
-
563
- print(f" ✅ 选择第{row_idx+1}行第{col_idx+1}列: 中心点({center_x}, {center_y})", file=sys.stderr)
564
-
565
- return {
566
- 'element': query,
567
- 'ref': bounds,
568
- 'confidence': 90,
569
- 'method': 'position_analysis',
570
- 'x': center_x,
571
- 'y': center_y,
572
- }
573
-
574
- # ========================================
575
- # 辅助方法
576
- # ========================================
577
-
578
- def _filter_by_region(self, elements: List[Dict], region: str) -> List[Dict]:
579
- """
580
- 按区域筛选元素
581
-
582
- Args:
583
- elements: 所有元素
584
- region: 区域名称('top', 'bottom', 'left', 'right')
585
-
586
- Returns:
587
- 筛选后的元素列表
588
- """
589
- if region not in self.regions:
590
- return elements
591
-
592
- region_range = self.regions[region]
593
- filtered = []
594
-
595
- for elem in elements:
596
- bounds = elem.get('bounds', '')
597
- if not bounds:
598
- continue
599
-
600
- center_x, center_y = self._get_center(elem)
601
-
602
- if region in ['top', 'bottom']:
603
- # 按Y坐标筛选
604
- if region_range[0] <= center_y <= region_range[1]:
605
- filtered.append(elem)
606
- elif region in ['left', 'right']:
607
- # 按X坐标筛选
608
- if region_range[0] <= center_x <= region_range[1]:
609
- filtered.append(elem)
610
-
611
- return filtered
612
-
613
- def _get_bounds(self, element: Dict) -> Optional[Tuple[int, int, int, int]]:
614
- """
615
- 解析bounds字符串
616
-
617
- Args:
618
- element: 元素信息
619
-
620
- Returns:
621
- (x1, y1, x2, y2) 或 None
622
- """
623
- bounds = element.get('bounds', '')
624
- if not bounds:
625
- return None
626
-
627
- # bounds格式: "[x1,y1][x2,y2]"
628
- match = re.search(r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]', bounds)
629
- if match:
630
- x1, y1, x2, y2 = map(int, match.groups())
631
- return (x1, y1, x2, y2)
632
-
633
- return None
634
-
635
- def _get_center(self, element: Dict) -> Tuple[int, int]:
636
- """
637
- 获取元素中心点坐标
638
-
639
- Args:
640
- element: 元素信息
641
-
642
- Returns:
643
- (center_x, center_y)
644
- """
645
- bounds = self._get_bounds(element)
646
- if bounds:
647
- x1, y1, x2, y2 = bounds
648
- center_x = (x1 + x2) // 2
649
- center_y = (y1 + y2) // 2
650
- return (center_x, center_y)
651
-
652
- return (0, 0)
653
-
654
- def _get_center_x(self, element: Dict) -> int:
655
- """获取元素中心点X坐标"""
656
- return self._get_center(element)[0]
657
-
658
- def _get_center_y(self, element: Dict) -> int:
659
- """获取元素中心点Y坐标"""
660
- return self._get_center(element)[1]
661
-
662
- def _is_system_ui(self, elem: Dict) -> bool:
663
- """
664
- 判断是否是系统UI元素(状态栏、导航栏等)
665
-
666
- Args:
667
- elem: 元素字典
668
-
669
- Returns:
670
- True if system UI, False otherwise
671
- """
672
- resource_id = elem.get('resource_id', '')
673
- class_name = elem.get('class_name', '')
674
-
675
- # 系统UI的resource-id通常以这些开头
676
- system_prefixes = [
677
- 'com.android.systemui',
678
- 'android:id/statusBarBackground',
679
- 'android:id/navigationBarBackground',
680
- ]
681
-
682
- return any(resource_id.startswith(prefix) for prefix in system_prefixes)
683
-
684
- def _extract_index(self, query: str) -> Optional[int]:
685
- """
686
- 从查询中提取索引
687
-
688
- Args:
689
- query: 查询文本(支持"第一个"、"第1个"等)
690
-
691
- Returns:
692
- 索引(1-based)或 None
693
- """
694
- # 中文数字映射
695
- chinese_numbers = {
696
- '一': 1, '二': 2, '三': 3, '四': 4, '五': 5,
697
- '六': 6, '七': 7, '八': 8, '九': 9, '十': 10,
698
- '1': 1, '2': 2, '3': 3, '4': 4, '5': 5,
699
- '6': 6, '7': 7, '8': 8, '9': 9, '0': 10,
700
- }
701
-
702
- # 匹配"第X个"、"第X项"、"第X列"等(支持中文数字)
703
- patterns = [
704
- r'第([一二三四五六七八九十\d]+)个',
705
- r'第([一二三四五六七八九十\d]+)项',
706
- r'第([一二三四五六七八九十\d]+)列',
707
- r'([一二三四五六七八九十\d]+)号',
708
- ]
709
-
710
- for pattern in patterns:
711
- match = re.search(pattern, query)
712
- if match:
713
- num_str = match.group(1)
714
- # 尝试转换为数字
715
- if num_str.isdigit():
716
- return int(num_str)
717
- elif num_str in chinese_numbers:
718
- return chinese_numbers[num_str]
719
-
720
- return None
721
-
722
- def _extract_grid_index(self, query: str) -> Tuple[Optional[int], Optional[int]]:
723
- """
724
- 从查询中提取网格索引
725
-
726
- Args:
727
- query: 查询文本(如"第2行第3列")
728
-
729
- Returns:
730
- (row_index, col_index) 或 (None, None)
731
- """
732
- # 匹配"第X行第Y列"
733
- match = re.search(r'第(\d+)行第(\d+)列', query)
734
- if match:
735
- row = int(match.group(1)) - 1 # 转换为0-based
736
- col = int(match.group(2)) - 1
737
- return (row, col)
738
-
739
- return (None, None)
740
-
741
- def _group_by_y(self, elements: List[Dict], num_groups: int) -> List[List[Dict]]:
742
- """
743
- 按Y坐标分组
744
-
745
- Args:
746
- elements: 元素列表
747
- num_groups: 分组数量
748
-
749
- Returns:
750
- 分组后的元素列表
751
- """
752
- # 按Y坐标排序
753
- sorted_elements = sorted(elements, key=lambda e: self._get_center_y(e))
754
-
755
- # 平均分组
756
- group_size = len(sorted_elements) // num_groups
757
- groups = []
758
-
759
- for i in range(num_groups):
760
- start = i * group_size
761
- end = start + group_size if i < num_groups - 1 else len(sorted_elements)
762
- groups.append(sorted_elements[start:end])
763
-
764
- return groups
765
-
766
- def _match_by_keyword(self, elements: List[Dict], query: str) -> Optional[Dict]:
767
- """
768
- 通过关键词匹配元素
769
-
770
- Args:
771
- elements: 候选元素列表(已排序)
772
- query: 查询文本
773
-
774
- Returns:
775
- 匹配的元素信息
776
- """
777
- # 关键词映射(可扩展)
778
- keyword_map = {
779
- '首页': 0,
780
- 'home': 0,
781
- '发现': 1,
782
- 'discover': 1,
783
- '社区': 2,
784
- 'community': 2,
785
- '我的': 3,
786
- 'profile': 3,
787
- '个人': 3,
788
- }
789
-
790
- query_lower = query.lower()
791
-
792
- for keyword, index in keyword_map.items():
793
- if keyword in query_lower:
794
- if index < len(elements):
795
- selected = elements[index]
796
- bounds = selected.get('bounds', '')
797
- center_x, center_y = self._get_center(selected)
798
-
799
- print(f" ✅ 关键词匹配: '{keyword}' → 第{index+1}个元素", file=sys.stderr)
800
- print(f" 中心点: ({center_x}, {center_y})", file=sys.stderr)
801
-
802
- return {
803
- 'element': query,
804
- 'ref': bounds,
805
- 'confidence': 90,
806
- 'method': 'position_analysis_keyword',
807
- 'x': center_x,
808
- 'y': center_y,
809
- }
810
-
811
- print(f" ❌ 未找到匹配的关键词", file=sys.stderr)
812
- return None
813
-