kotonebot 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kotonebot/__init__.py +39 -39
- kotonebot/backend/bot.py +312 -312
- kotonebot/backend/color.py +525 -525
- kotonebot/backend/context/__init__.py +3 -3
- kotonebot/backend/context/context.py +1002 -1002
- kotonebot/backend/context/task_action.py +183 -183
- kotonebot/backend/core.py +86 -129
- kotonebot/backend/debug/entry.py +89 -89
- kotonebot/backend/debug/mock.py +78 -78
- kotonebot/backend/debug/server.py +222 -222
- kotonebot/backend/debug/vars.py +351 -351
- kotonebot/backend/dispatch.py +227 -227
- kotonebot/backend/flow_controller.py +196 -196
- kotonebot/backend/image.py +36 -5
- kotonebot/backend/loop.py +222 -208
- kotonebot/backend/ocr.py +535 -535
- kotonebot/backend/preprocessor.py +103 -103
- kotonebot/client/__init__.py +9 -9
- kotonebot/client/device.py +369 -529
- kotonebot/client/fast_screenshot.py +377 -377
- kotonebot/client/host/__init__.py +43 -43
- kotonebot/client/host/adb_common.py +101 -107
- kotonebot/client/host/custom.py +118 -118
- kotonebot/client/host/leidian_host.py +196 -196
- kotonebot/client/host/mumu12_host.py +353 -353
- kotonebot/client/host/protocol.py +214 -214
- kotonebot/client/host/windows_common.py +73 -58
- kotonebot/client/implements/__init__.py +65 -70
- kotonebot/client/implements/adb.py +89 -89
- kotonebot/client/implements/nemu_ipc/__init__.py +11 -11
- kotonebot/client/implements/nemu_ipc/external_renderer_ipc.py +284 -284
- kotonebot/client/implements/nemu_ipc/nemu_ipc.py +327 -327
- kotonebot/client/implements/remote_windows.py +188 -188
- kotonebot/client/implements/uiautomator2.py +85 -85
- kotonebot/client/implements/windows/__init__.py +1 -0
- kotonebot/client/implements/windows/print_window.py +133 -0
- kotonebot/client/implements/windows/send_message.py +324 -0
- kotonebot/client/implements/{windows.py → windows/windows.py} +175 -176
- kotonebot/client/protocol.py +69 -69
- kotonebot/client/registration.py +24 -24
- kotonebot/client/scaler.py +467 -0
- kotonebot/config/base_config.py +103 -96
- kotonebot/config/config.py +61 -0
- kotonebot/config/manager.py +36 -36
- kotonebot/core/__init__.py +13 -0
- kotonebot/core/entities/base.py +182 -0
- kotonebot/core/entities/compound.py +75 -0
- kotonebot/core/entities/ocr.py +117 -0
- kotonebot/core/entities/template_match.py +198 -0
- kotonebot/devtools/__init__.py +42 -0
- kotonebot/devtools/cli/__init__.py +6 -0
- kotonebot/devtools/cli/main.py +53 -0
- kotonebot/{tools → devtools}/mirror.py +354 -354
- kotonebot/devtools/project/project.py +41 -0
- kotonebot/devtools/project/scanner.py +202 -0
- kotonebot/devtools/project/schema.py +99 -0
- kotonebot/devtools/resgen/__init__.py +42 -0
- kotonebot/devtools/resgen/codegen.py +331 -0
- kotonebot/devtools/resgen/core.py +94 -0
- kotonebot/devtools/resgen/parsers.py +360 -0
- kotonebot/devtools/resgen/utils.py +158 -0
- kotonebot/devtools/resgen/validation.py +115 -0
- kotonebot/devtools/web/dist/assets/bootstrap-icons-BOrJxbIo.woff +0 -0
- kotonebot/devtools/web/dist/assets/bootstrap-icons-BtvjY1KL.woff2 +0 -0
- kotonebot/devtools/web/dist/assets/ext-language_tools-CD021WJ2.js +2577 -0
- kotonebot/devtools/web/dist/assets/index-B_m5f2LF.js +2836 -0
- kotonebot/devtools/web/dist/assets/index-BlEDyGGa.css +9 -0
- kotonebot/devtools/web/dist/assets/language-client-C9muzqaq.js +128 -0
- kotonebot/devtools/web/dist/assets/mode-python-CtHp76XS.js +476 -0
- kotonebot/devtools/web/dist/icons/symbol-class.svg +3 -0
- kotonebot/devtools/web/dist/icons/symbol-file.svg +3 -0
- kotonebot/devtools/web/dist/icons/symbol-method.svg +3 -0
- kotonebot/devtools/web/dist/index.html +25 -0
- kotonebot/devtools/web/server/__init__.py +0 -0
- kotonebot/devtools/web/server/rest_api.py +217 -0
- kotonebot/devtools/web/server/server.py +85 -0
- kotonebot/errors.py +76 -76
- kotonebot/interop/win/__init__.py +13 -9
- kotonebot/interop/win/_mouse.py +310 -310
- kotonebot/interop/win/message_box.py +313 -313
- kotonebot/interop/win/reg.py +37 -37
- kotonebot/interop/win/shake_mouse.py +224 -0
- kotonebot/interop/win/shortcut.py +43 -43
- kotonebot/interop/win/task_dialog.py +513 -513
- kotonebot/interop/win/window.py +89 -0
- kotonebot/logging/__init__.py +2 -2
- kotonebot/logging/log.py +17 -17
- kotonebot/primitives/__init__.py +19 -17
- kotonebot/primitives/geometry.py +1067 -862
- kotonebot/primitives/visual.py +143 -63
- kotonebot/ui/file_host/sensio.py +36 -36
- kotonebot/ui/file_host/tmp_send.py +54 -54
- kotonebot/ui/pushkit/__init__.py +3 -3
- kotonebot/ui/pushkit/image_host.py +88 -88
- kotonebot/ui/pushkit/protocol.py +13 -13
- kotonebot/ui/pushkit/wxpusher.py +54 -54
- kotonebot/ui/user.py +148 -148
- kotonebot/util.py +436 -436
- {kotonebot-0.5.0.dist-info → kotonebot-0.7.0.dist-info}/METADATA +84 -82
- kotonebot-0.7.0.dist-info/RECORD +109 -0
- {kotonebot-0.5.0.dist-info → kotonebot-0.7.0.dist-info}/WHEEL +1 -1
- kotonebot-0.7.0.dist-info/entry_points.txt +2 -0
- {kotonebot-0.5.0.dist-info → kotonebot-0.7.0.dist-info}/licenses/LICENSE +673 -673
- kotonebot/client/implements/adb_raw.py +0 -163
- kotonebot-0.5.0.dist-info/RECORD +0 -71
- /kotonebot/{tools → devtools/project}/__init__.py +0 -0
- {kotonebot-0.5.0.dist-info → kotonebot-0.7.0.dist-info}/top_level.txt +0 -0
|
@@ -1,354 +1,354 @@
|
|
|
1
|
-
import wx
|
|
2
|
-
import cv2
|
|
3
|
-
import numpy as np
|
|
4
|
-
import time
|
|
5
|
-
from typing import Optional, Tuple, Callable
|
|
6
|
-
from threading import Thread, Lock
|
|
7
|
-
from cv2.typing import MatLike
|
|
8
|
-
from queue import Queue
|
|
9
|
-
|
|
10
|
-
from kotonebot.client.device import Device
|
|
11
|
-
|
|
12
|
-
class DeviceMirrorPanel(wx.Panel):
|
|
13
|
-
def __init__(self, parent, device: Device, log_callback=None):
|
|
14
|
-
super().__init__(parent)
|
|
15
|
-
self.device = device
|
|
16
|
-
self.screen_bitmap: Optional[wx.Bitmap] = None
|
|
17
|
-
self.fps = 0
|
|
18
|
-
self.last_frame_time = time.time()
|
|
19
|
-
self.frame_count = 0
|
|
20
|
-
self.is_running = True
|
|
21
|
-
self.lock = Lock()
|
|
22
|
-
self.last_mouse_pos = (0, 0)
|
|
23
|
-
self.is_dragging = False
|
|
24
|
-
self.screenshot_interval = 0 # 截图耗时(ms)
|
|
25
|
-
self.log_callback = log_callback
|
|
26
|
-
self.operation_queue = Queue()
|
|
27
|
-
|
|
28
|
-
# 设置背景色为黑色
|
|
29
|
-
self.SetBackgroundColour(wx.BLACK)
|
|
30
|
-
|
|
31
|
-
# 双缓冲,减少闪烁
|
|
32
|
-
self.SetDoubleBuffered(True)
|
|
33
|
-
|
|
34
|
-
# 绑定事件
|
|
35
|
-
self.Bind(wx.EVT_PAINT, self.on_paint)
|
|
36
|
-
self.Bind(wx.EVT_SIZE, self.on_size)
|
|
37
|
-
self.Bind(wx.EVT_LEFT_DOWN, self.on_left_down)
|
|
38
|
-
self.Bind(wx.EVT_LEFT_UP, self.on_left_up)
|
|
39
|
-
self.Bind(wx.EVT_MOTION, self.on_motion)
|
|
40
|
-
|
|
41
|
-
# 启动刷新线程
|
|
42
|
-
self.update_thread = Thread(target=self.update_screen, daemon=True)
|
|
43
|
-
self.update_thread.start()
|
|
44
|
-
|
|
45
|
-
# 启动操作处理线程
|
|
46
|
-
self.operation_thread = Thread(target=self.process_operations, daemon=True)
|
|
47
|
-
self.operation_thread.start()
|
|
48
|
-
|
|
49
|
-
def process_operations(self):
|
|
50
|
-
"""处理设备操作的线程"""
|
|
51
|
-
while self.is_running:
|
|
52
|
-
try:
|
|
53
|
-
operation = self.operation_queue.get()
|
|
54
|
-
if operation is not None:
|
|
55
|
-
operation()
|
|
56
|
-
self.operation_queue.task_done()
|
|
57
|
-
except Exception as e:
|
|
58
|
-
if self.log_callback:
|
|
59
|
-
self.log_callback(f"操作执行错误: {e}")
|
|
60
|
-
|
|
61
|
-
def execute_device_operation(self, operation: Callable):
|
|
62
|
-
"""将设备操作添加到队列"""
|
|
63
|
-
self.operation_queue.put(operation)
|
|
64
|
-
|
|
65
|
-
def update_screen(self):
|
|
66
|
-
while self.is_running:
|
|
67
|
-
try:
|
|
68
|
-
# 获取设备截图并计时
|
|
69
|
-
start_time = time.time()
|
|
70
|
-
frame = self.device.screenshot()
|
|
71
|
-
end_time = time.time()
|
|
72
|
-
self.screenshot_interval = int((end_time - start_time) * 1000)
|
|
73
|
-
|
|
74
|
-
if frame is None:
|
|
75
|
-
continue
|
|
76
|
-
|
|
77
|
-
# 计算FPS
|
|
78
|
-
current_time = time.time()
|
|
79
|
-
self.frame_count += 1
|
|
80
|
-
if current_time - self.last_frame_time >= 1.0:
|
|
81
|
-
self.fps = self.frame_count
|
|
82
|
-
self.frame_count = 0
|
|
83
|
-
self.last_frame_time = current_time
|
|
84
|
-
|
|
85
|
-
# 转换为wx.Bitmap
|
|
86
|
-
height, width = frame.shape[:2]
|
|
87
|
-
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
88
|
-
wximage = wx.Bitmap.FromBuffer(width, height, frame)
|
|
89
|
-
|
|
90
|
-
with self.lock:
|
|
91
|
-
self.screen_bitmap = wximage
|
|
92
|
-
|
|
93
|
-
# 请求重绘
|
|
94
|
-
wx.CallAfter(self.Refresh)
|
|
95
|
-
|
|
96
|
-
# 控制刷新率
|
|
97
|
-
time.sleep(1/60)
|
|
98
|
-
|
|
99
|
-
except Exception as e:
|
|
100
|
-
print(f"Error updating screen: {e}")
|
|
101
|
-
time.sleep(1)
|
|
102
|
-
|
|
103
|
-
def on_paint(self, event):
|
|
104
|
-
dc = wx.BufferedPaintDC(self)
|
|
105
|
-
|
|
106
|
-
# 清空背景
|
|
107
|
-
dc.SetBackground(wx.Brush(wx.BLACK))
|
|
108
|
-
dc.Clear()
|
|
109
|
-
|
|
110
|
-
if not self.screen_bitmap:
|
|
111
|
-
return
|
|
112
|
-
|
|
113
|
-
# 绘制设备画面
|
|
114
|
-
with self.lock:
|
|
115
|
-
# 计算缩放比例,保持宽高比
|
|
116
|
-
panel_width, panel_height = self.GetSize()
|
|
117
|
-
bitmap_width = self.screen_bitmap.GetWidth()
|
|
118
|
-
bitmap_height = self.screen_bitmap.GetHeight()
|
|
119
|
-
|
|
120
|
-
scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
|
|
121
|
-
scaled_width = int(bitmap_width * scale)
|
|
122
|
-
scaled_height = int(bitmap_height * scale)
|
|
123
|
-
|
|
124
|
-
# 居中显示
|
|
125
|
-
x = (panel_width - scaled_width) // 2
|
|
126
|
-
y = (panel_height - scaled_height) // 2
|
|
127
|
-
|
|
128
|
-
if scale != 1:
|
|
129
|
-
img = self.screen_bitmap.ConvertToImage()
|
|
130
|
-
img = img.Scale(scaled_width, scaled_height, wx.IMAGE_QUALITY_HIGH)
|
|
131
|
-
bitmap = wx.Bitmap(img)
|
|
132
|
-
else:
|
|
133
|
-
bitmap = self.screen_bitmap
|
|
134
|
-
|
|
135
|
-
dc.DrawBitmap(bitmap, x, y)
|
|
136
|
-
|
|
137
|
-
# 绘制FPS和截图时间
|
|
138
|
-
dc.SetTextForeground(wx.GREEN)
|
|
139
|
-
dc.SetFont(wx.Font(10, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD))
|
|
140
|
-
dc.DrawText(f"FPS: {self.fps}", 10, 10)
|
|
141
|
-
dc.DrawText(f"Interval: {self.screenshot_interval}ms", 10, 30)
|
|
142
|
-
|
|
143
|
-
def on_size(self, event):
|
|
144
|
-
self.Refresh()
|
|
145
|
-
event.Skip()
|
|
146
|
-
|
|
147
|
-
def get_device_coordinates(self, x: int, y: int) -> Tuple[int, int]:
|
|
148
|
-
"""将面板坐标转换为设备坐标"""
|
|
149
|
-
if not self.screen_bitmap:
|
|
150
|
-
return (0, 0)
|
|
151
|
-
|
|
152
|
-
panel_width, panel_height = self.GetSize()
|
|
153
|
-
bitmap_width = self.screen_bitmap.GetWidth()
|
|
154
|
-
bitmap_height = self.screen_bitmap.GetHeight()
|
|
155
|
-
|
|
156
|
-
scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
|
|
157
|
-
scaled_width = int(bitmap_width * scale)
|
|
158
|
-
scaled_height = int(bitmap_height * scale)
|
|
159
|
-
|
|
160
|
-
# 计算显示区域的偏移
|
|
161
|
-
x_offset = (panel_width - scaled_width) // 2
|
|
162
|
-
y_offset = (panel_height - scaled_height) // 2
|
|
163
|
-
|
|
164
|
-
# 转换坐标
|
|
165
|
-
device_x = int((x - x_offset) / scale)
|
|
166
|
-
device_y = int((y - y_offset) / scale)
|
|
167
|
-
|
|
168
|
-
# 确保坐标在设备范围内
|
|
169
|
-
device_x = max(0, min(device_x, bitmap_width-1))
|
|
170
|
-
device_y = max(0, min(device_y, bitmap_height-1))
|
|
171
|
-
|
|
172
|
-
return (device_x, device_y)
|
|
173
|
-
|
|
174
|
-
def on_left_down(self, event):
|
|
175
|
-
self.last_mouse_pos = event.GetPosition()
|
|
176
|
-
self.is_dragging = True
|
|
177
|
-
event.Skip()
|
|
178
|
-
|
|
179
|
-
def on_left_up(self, event):
|
|
180
|
-
if not self.is_dragging:
|
|
181
|
-
return
|
|
182
|
-
|
|
183
|
-
self.is_dragging = False
|
|
184
|
-
pos = event.GetPosition()
|
|
185
|
-
|
|
186
|
-
# 如果鼠标位置没有明显变化,执行点击
|
|
187
|
-
if abs(pos[0] - self.last_mouse_pos[0]) < 5 and abs(pos[1] - self.last_mouse_pos[1]) < 5:
|
|
188
|
-
device_x, device_y = self.get_device_coordinates(*pos)
|
|
189
|
-
self.execute_device_operation(lambda: self.device.click(device_x, device_y))
|
|
190
|
-
if self.log_callback:
|
|
191
|
-
self.log_callback(f"点击: ({device_x}, {device_y})")
|
|
192
|
-
else:
|
|
193
|
-
# 执行滑动
|
|
194
|
-
start_x, start_y = self.get_device_coordinates(*self.last_mouse_pos)
|
|
195
|
-
end_x, end_y = self.get_device_coordinates(*pos)
|
|
196
|
-
self.execute_device_operation(lambda: self.device.swipe(start_x, start_y, end_x, end_y))
|
|
197
|
-
if self.log_callback:
|
|
198
|
-
self.log_callback(f"滑动: ({start_x}, {start_y}) -> ({end_x}, {end_y})")
|
|
199
|
-
|
|
200
|
-
event.Skip()
|
|
201
|
-
|
|
202
|
-
def on_motion(self, event):
|
|
203
|
-
if not self.is_dragging:
|
|
204
|
-
event.Skip()
|
|
205
|
-
return
|
|
206
|
-
|
|
207
|
-
event.Skip()
|
|
208
|
-
|
|
209
|
-
class DeviceMirrorFrame(wx.Frame):
|
|
210
|
-
def __init__(self, device: Device):
|
|
211
|
-
super().__init__(None, title="设备镜像", size=(800, 600))
|
|
212
|
-
|
|
213
|
-
# 创建分割窗口
|
|
214
|
-
self.splitter = wx.SplitterWindow(self)
|
|
215
|
-
|
|
216
|
-
# 创建左侧面板(包含控制区域和日志区域)
|
|
217
|
-
self.left_panel = wx.Panel(self.splitter)
|
|
218
|
-
left_sizer = wx.BoxSizer(wx.VERTICAL)
|
|
219
|
-
|
|
220
|
-
# 控制区域
|
|
221
|
-
self.control_panel = wx.Panel(self.left_panel)
|
|
222
|
-
self.init_control_panel()
|
|
223
|
-
left_sizer.Add(self.control_panel, 0, wx.EXPAND | wx.ALL, 5)
|
|
224
|
-
|
|
225
|
-
# 日志区域
|
|
226
|
-
self.log_text = wx.TextCtrl(self.left_panel, style=wx.TE_MULTILINE | wx.TE_READONLY | wx.HSCROLL)
|
|
227
|
-
self.log_text.SetBackgroundColour(wx.BLACK)
|
|
228
|
-
self.log_text.SetForegroundColour(wx.GREEN)
|
|
229
|
-
self.log_text.SetFont(wx.Font(9, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL))
|
|
230
|
-
left_sizer.Add(self.log_text, 1, wx.EXPAND | wx.ALL, 5)
|
|
231
|
-
|
|
232
|
-
self.left_panel.SetSizer(left_sizer)
|
|
233
|
-
|
|
234
|
-
# 创建设备画面
|
|
235
|
-
self.device_panel = DeviceMirrorPanel(self.splitter, device, self.log)
|
|
236
|
-
|
|
237
|
-
# 设置分割
|
|
238
|
-
self.splitter.SplitVertically(self.left_panel, self.device_panel)
|
|
239
|
-
self.splitter.SetMinimumPaneSize(200)
|
|
240
|
-
|
|
241
|
-
# 保存设备引用
|
|
242
|
-
self.device = device
|
|
243
|
-
|
|
244
|
-
def log(self, message: str):
|
|
245
|
-
"""添加日志"""
|
|
246
|
-
timestamp = time.strftime("%H:%M:%S", time.localtime())
|
|
247
|
-
wx.CallAfter(self.log_text.AppendText, f"[{timestamp}] {message}\n")
|
|
248
|
-
|
|
249
|
-
def init_control_panel(self):
|
|
250
|
-
vbox = wx.BoxSizer(wx.VERTICAL)
|
|
251
|
-
|
|
252
|
-
# 添加控制按钮
|
|
253
|
-
btn_get_resolution = wx.Button(self.control_panel, label="获取分辨率")
|
|
254
|
-
btn_get_resolution.Bind(wx.EVT_BUTTON, self.on_get_resolution)
|
|
255
|
-
vbox.Add(btn_get_resolution, 0, wx.EXPAND | wx.ALL, 5)
|
|
256
|
-
|
|
257
|
-
btn_get_orientation = wx.Button(self.control_panel, label="获取设备方向")
|
|
258
|
-
btn_get_orientation.Bind(wx.EVT_BUTTON, self.on_get_orientation)
|
|
259
|
-
vbox.Add(btn_get_orientation, 0, wx.EXPAND | wx.ALL, 5)
|
|
260
|
-
|
|
261
|
-
# 启动APP区域
|
|
262
|
-
hbox = wx.BoxSizer(wx.HORIZONTAL)
|
|
263
|
-
self.package_input = wx.TextCtrl(self.control_panel)
|
|
264
|
-
hbox.Add(self.package_input, 1, wx.EXPAND | wx.RIGHT, 5)
|
|
265
|
-
btn_launch_app = wx.Button(self.control_panel, label="启动APP")
|
|
266
|
-
btn_launch_app.Bind(wx.EVT_BUTTON, self.on_launch_app)
|
|
267
|
-
hbox.Add(btn_launch_app, 0)
|
|
268
|
-
vbox.Add(hbox, 0, wx.EXPAND | wx.ALL, 5)
|
|
269
|
-
|
|
270
|
-
btn_get_current_app = wx.Button(self.control_panel, label="获取前台APP")
|
|
271
|
-
btn_get_current_app.Bind(wx.EVT_BUTTON, self.on_get_current_app)
|
|
272
|
-
vbox.Add(btn_get_current_app, 0, wx.EXPAND | wx.ALL, 5)
|
|
273
|
-
|
|
274
|
-
self.control_panel.SetSizer(vbox)
|
|
275
|
-
|
|
276
|
-
def on_get_resolution(self, event):
|
|
277
|
-
"""获取分辨率"""
|
|
278
|
-
try:
|
|
279
|
-
width, height = self.device.screen_size
|
|
280
|
-
self.log(f"设备分辨率: {width}x{height}")
|
|
281
|
-
except Exception as e:
|
|
282
|
-
self.log(f"获取分辨率失败: {e}")
|
|
283
|
-
|
|
284
|
-
def on_get_orientation(self, event):
|
|
285
|
-
"""获取设备方向"""
|
|
286
|
-
try:
|
|
287
|
-
orientation = self.device.detect_orientation()
|
|
288
|
-
orientation_text = "横屏" if orientation == "landscape" else "竖屏"
|
|
289
|
-
self.log(f"设备方向: {orientation_text}")
|
|
290
|
-
except Exception as e:
|
|
291
|
-
self.log(f"获取设备方向失败: {e}")
|
|
292
|
-
|
|
293
|
-
def on_launch_app(self, event):
|
|
294
|
-
"""启动APP"""
|
|
295
|
-
package_name = self.package_input.GetValue().strip()
|
|
296
|
-
if not package_name:
|
|
297
|
-
self.log("请输入包名")
|
|
298
|
-
return
|
|
299
|
-
try:
|
|
300
|
-
# 使用新的 API 通过 commands 属性访问平台特定方法
|
|
301
|
-
if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'launch_app'):
|
|
302
|
-
self.device.commands.launch_app(package_name)
|
|
303
|
-
self.log(f"启动APP: {package_name}")
|
|
304
|
-
else:
|
|
305
|
-
self.log("当前设备不支持启动APP功能")
|
|
306
|
-
except Exception as e:
|
|
307
|
-
self.log(f"启动APP失败: {e}")
|
|
308
|
-
|
|
309
|
-
def on_get_current_app(self, event):
|
|
310
|
-
"""获取前台APP"""
|
|
311
|
-
try:
|
|
312
|
-
# 使用新的 API 通过 commands 属性访问平台特定方法
|
|
313
|
-
if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'current_package'):
|
|
314
|
-
package = self.device.commands.current_package()
|
|
315
|
-
if package:
|
|
316
|
-
self.log(f"前台APP: {package}")
|
|
317
|
-
else:
|
|
318
|
-
self.log("未获取到前台APP")
|
|
319
|
-
else:
|
|
320
|
-
self.log("当前设备不支持获取前台APP功能")
|
|
321
|
-
except Exception as e:
|
|
322
|
-
self.log(f"获取前台APP失败: {e}")
|
|
323
|
-
|
|
324
|
-
def on_quit(self, event):
|
|
325
|
-
self.device_panel.is_running = False
|
|
326
|
-
self.Close()
|
|
327
|
-
|
|
328
|
-
def show_device_mirror(device: Device):
|
|
329
|
-
"""显示设备镜像窗口"""
|
|
330
|
-
app = wx.App()
|
|
331
|
-
frame = DeviceMirrorFrame(device)
|
|
332
|
-
frame.Show()
|
|
333
|
-
app.MainLoop()
|
|
334
|
-
|
|
335
|
-
if __name__ == "__main__":
|
|
336
|
-
# 测试代码
|
|
337
|
-
from kotonebot.client.device import AndroidDevice
|
|
338
|
-
from kotonebot.client.implements.adb import AdbImpl
|
|
339
|
-
from kotonebot.client.implements.uiautomator2 import UiAutomator2Impl
|
|
340
|
-
from adbutils import adb
|
|
341
|
-
|
|
342
|
-
print("server version:", adb.server_version())
|
|
343
|
-
adb.connect("127.0.0.1:5555")
|
|
344
|
-
print("devices:", adb.device_list())
|
|
345
|
-
d = adb.device_list()[-1]
|
|
346
|
-
|
|
347
|
-
# 使用新的 API
|
|
348
|
-
dd = AndroidDevice(d)
|
|
349
|
-
adb_imp = AdbImpl(d) # 直接传入 adb 连接
|
|
350
|
-
dd._touch = adb_imp
|
|
351
|
-
dd._screenshot = UiAutomator2Impl(dd) # UiAutomator2Impl 可能还需要 device 对象
|
|
352
|
-
dd.commands = adb_imp # 设置 Android 特定命令
|
|
353
|
-
|
|
354
|
-
show_device_mirror(dd)
|
|
1
|
+
import wx
|
|
2
|
+
import cv2
|
|
3
|
+
import numpy as np
|
|
4
|
+
import time
|
|
5
|
+
from typing import Optional, Tuple, Callable
|
|
6
|
+
from threading import Thread, Lock
|
|
7
|
+
from cv2.typing import MatLike
|
|
8
|
+
from queue import Queue
|
|
9
|
+
|
|
10
|
+
from kotonebot.client.device import Device
|
|
11
|
+
|
|
12
|
+
class DeviceMirrorPanel(wx.Panel):
|
|
13
|
+
def __init__(self, parent, device: Device, log_callback=None):
|
|
14
|
+
super().__init__(parent)
|
|
15
|
+
self.device = device
|
|
16
|
+
self.screen_bitmap: Optional[wx.Bitmap] = None
|
|
17
|
+
self.fps = 0
|
|
18
|
+
self.last_frame_time = time.time()
|
|
19
|
+
self.frame_count = 0
|
|
20
|
+
self.is_running = True
|
|
21
|
+
self.lock = Lock()
|
|
22
|
+
self.last_mouse_pos = (0, 0)
|
|
23
|
+
self.is_dragging = False
|
|
24
|
+
self.screenshot_interval = 0 # 截图耗时(ms)
|
|
25
|
+
self.log_callback = log_callback
|
|
26
|
+
self.operation_queue = Queue()
|
|
27
|
+
|
|
28
|
+
# 设置背景色为黑色
|
|
29
|
+
self.SetBackgroundColour(wx.BLACK)
|
|
30
|
+
|
|
31
|
+
# 双缓冲,减少闪烁
|
|
32
|
+
self.SetDoubleBuffered(True)
|
|
33
|
+
|
|
34
|
+
# 绑定事件
|
|
35
|
+
self.Bind(wx.EVT_PAINT, self.on_paint)
|
|
36
|
+
self.Bind(wx.EVT_SIZE, self.on_size)
|
|
37
|
+
self.Bind(wx.EVT_LEFT_DOWN, self.on_left_down)
|
|
38
|
+
self.Bind(wx.EVT_LEFT_UP, self.on_left_up)
|
|
39
|
+
self.Bind(wx.EVT_MOTION, self.on_motion)
|
|
40
|
+
|
|
41
|
+
# 启动刷新线程
|
|
42
|
+
self.update_thread = Thread(target=self.update_screen, daemon=True)
|
|
43
|
+
self.update_thread.start()
|
|
44
|
+
|
|
45
|
+
# 启动操作处理线程
|
|
46
|
+
self.operation_thread = Thread(target=self.process_operations, daemon=True)
|
|
47
|
+
self.operation_thread.start()
|
|
48
|
+
|
|
49
|
+
def process_operations(self):
|
|
50
|
+
"""处理设备操作的线程"""
|
|
51
|
+
while self.is_running:
|
|
52
|
+
try:
|
|
53
|
+
operation = self.operation_queue.get()
|
|
54
|
+
if operation is not None:
|
|
55
|
+
operation()
|
|
56
|
+
self.operation_queue.task_done()
|
|
57
|
+
except Exception as e:
|
|
58
|
+
if self.log_callback:
|
|
59
|
+
self.log_callback(f"操作执行错误: {e}")
|
|
60
|
+
|
|
61
|
+
def execute_device_operation(self, operation: Callable):
|
|
62
|
+
"""将设备操作添加到队列"""
|
|
63
|
+
self.operation_queue.put(operation)
|
|
64
|
+
|
|
65
|
+
def update_screen(self):
|
|
66
|
+
while self.is_running:
|
|
67
|
+
try:
|
|
68
|
+
# 获取设备截图并计时
|
|
69
|
+
start_time = time.time()
|
|
70
|
+
frame = self.device.screenshot()
|
|
71
|
+
end_time = time.time()
|
|
72
|
+
self.screenshot_interval = int((end_time - start_time) * 1000)
|
|
73
|
+
|
|
74
|
+
if frame is None:
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
# 计算FPS
|
|
78
|
+
current_time = time.time()
|
|
79
|
+
self.frame_count += 1
|
|
80
|
+
if current_time - self.last_frame_time >= 1.0:
|
|
81
|
+
self.fps = self.frame_count
|
|
82
|
+
self.frame_count = 0
|
|
83
|
+
self.last_frame_time = current_time
|
|
84
|
+
|
|
85
|
+
# 转换为wx.Bitmap
|
|
86
|
+
height, width = frame.shape[:2]
|
|
87
|
+
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
88
|
+
wximage = wx.Bitmap.FromBuffer(width, height, frame)
|
|
89
|
+
|
|
90
|
+
with self.lock:
|
|
91
|
+
self.screen_bitmap = wximage
|
|
92
|
+
|
|
93
|
+
# 请求重绘
|
|
94
|
+
wx.CallAfter(self.Refresh)
|
|
95
|
+
|
|
96
|
+
# 控制刷新率
|
|
97
|
+
time.sleep(1/60)
|
|
98
|
+
|
|
99
|
+
except Exception as e:
|
|
100
|
+
print(f"Error updating screen: {e}")
|
|
101
|
+
time.sleep(1)
|
|
102
|
+
|
|
103
|
+
def on_paint(self, event):
|
|
104
|
+
dc = wx.BufferedPaintDC(self)
|
|
105
|
+
|
|
106
|
+
# 清空背景
|
|
107
|
+
dc.SetBackground(wx.Brush(wx.BLACK))
|
|
108
|
+
dc.Clear()
|
|
109
|
+
|
|
110
|
+
if not self.screen_bitmap:
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
# 绘制设备画面
|
|
114
|
+
with self.lock:
|
|
115
|
+
# 计算缩放比例,保持宽高比
|
|
116
|
+
panel_width, panel_height = self.GetSize()
|
|
117
|
+
bitmap_width = self.screen_bitmap.GetWidth()
|
|
118
|
+
bitmap_height = self.screen_bitmap.GetHeight()
|
|
119
|
+
|
|
120
|
+
scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
|
|
121
|
+
scaled_width = int(bitmap_width * scale)
|
|
122
|
+
scaled_height = int(bitmap_height * scale)
|
|
123
|
+
|
|
124
|
+
# 居中显示
|
|
125
|
+
x = (panel_width - scaled_width) // 2
|
|
126
|
+
y = (panel_height - scaled_height) // 2
|
|
127
|
+
|
|
128
|
+
if scale != 1:
|
|
129
|
+
img = self.screen_bitmap.ConvertToImage()
|
|
130
|
+
img = img.Scale(scaled_width, scaled_height, wx.IMAGE_QUALITY_HIGH)
|
|
131
|
+
bitmap = wx.Bitmap(img)
|
|
132
|
+
else:
|
|
133
|
+
bitmap = self.screen_bitmap
|
|
134
|
+
|
|
135
|
+
dc.DrawBitmap(bitmap, x, y)
|
|
136
|
+
|
|
137
|
+
# 绘制FPS和截图时间
|
|
138
|
+
dc.SetTextForeground(wx.GREEN)
|
|
139
|
+
dc.SetFont(wx.Font(10, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD))
|
|
140
|
+
dc.DrawText(f"FPS: {self.fps}", 10, 10)
|
|
141
|
+
dc.DrawText(f"Interval: {self.screenshot_interval}ms", 10, 30)
|
|
142
|
+
|
|
143
|
+
def on_size(self, event):
|
|
144
|
+
self.Refresh()
|
|
145
|
+
event.Skip()
|
|
146
|
+
|
|
147
|
+
def get_device_coordinates(self, x: int, y: int) -> Tuple[int, int]:
|
|
148
|
+
"""将面板坐标转换为设备坐标"""
|
|
149
|
+
if not self.screen_bitmap:
|
|
150
|
+
return (0, 0)
|
|
151
|
+
|
|
152
|
+
panel_width, panel_height = self.GetSize()
|
|
153
|
+
bitmap_width = self.screen_bitmap.GetWidth()
|
|
154
|
+
bitmap_height = self.screen_bitmap.GetHeight()
|
|
155
|
+
|
|
156
|
+
scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
|
|
157
|
+
scaled_width = int(bitmap_width * scale)
|
|
158
|
+
scaled_height = int(bitmap_height * scale)
|
|
159
|
+
|
|
160
|
+
# 计算显示区域的偏移
|
|
161
|
+
x_offset = (panel_width - scaled_width) // 2
|
|
162
|
+
y_offset = (panel_height - scaled_height) // 2
|
|
163
|
+
|
|
164
|
+
# 转换坐标
|
|
165
|
+
device_x = int((x - x_offset) / scale)
|
|
166
|
+
device_y = int((y - y_offset) / scale)
|
|
167
|
+
|
|
168
|
+
# 确保坐标在设备范围内
|
|
169
|
+
device_x = max(0, min(device_x, bitmap_width-1))
|
|
170
|
+
device_y = max(0, min(device_y, bitmap_height-1))
|
|
171
|
+
|
|
172
|
+
return (device_x, device_y)
|
|
173
|
+
|
|
174
|
+
def on_left_down(self, event):
|
|
175
|
+
self.last_mouse_pos = event.GetPosition()
|
|
176
|
+
self.is_dragging = True
|
|
177
|
+
event.Skip()
|
|
178
|
+
|
|
179
|
+
def on_left_up(self, event):
|
|
180
|
+
if not self.is_dragging:
|
|
181
|
+
return
|
|
182
|
+
|
|
183
|
+
self.is_dragging = False
|
|
184
|
+
pos = event.GetPosition()
|
|
185
|
+
|
|
186
|
+
# 如果鼠标位置没有明显变化,执行点击
|
|
187
|
+
if abs(pos[0] - self.last_mouse_pos[0]) < 5 and abs(pos[1] - self.last_mouse_pos[1]) < 5:
|
|
188
|
+
device_x, device_y = self.get_device_coordinates(*pos)
|
|
189
|
+
self.execute_device_operation(lambda: self.device.click(device_x, device_y))
|
|
190
|
+
if self.log_callback:
|
|
191
|
+
self.log_callback(f"点击: ({device_x}, {device_y})")
|
|
192
|
+
else:
|
|
193
|
+
# 执行滑动
|
|
194
|
+
start_x, start_y = self.get_device_coordinates(*self.last_mouse_pos)
|
|
195
|
+
end_x, end_y = self.get_device_coordinates(*pos)
|
|
196
|
+
self.execute_device_operation(lambda: self.device.swipe(start_x, start_y, end_x, end_y))
|
|
197
|
+
if self.log_callback:
|
|
198
|
+
self.log_callback(f"滑动: ({start_x}, {start_y}) -> ({end_x}, {end_y})")
|
|
199
|
+
|
|
200
|
+
event.Skip()
|
|
201
|
+
|
|
202
|
+
def on_motion(self, event):
|
|
203
|
+
if not self.is_dragging:
|
|
204
|
+
event.Skip()
|
|
205
|
+
return
|
|
206
|
+
|
|
207
|
+
event.Skip()
|
|
208
|
+
|
|
209
|
+
class DeviceMirrorFrame(wx.Frame):
|
|
210
|
+
def __init__(self, device: Device):
|
|
211
|
+
super().__init__(None, title="设备镜像", size=(800, 600))
|
|
212
|
+
|
|
213
|
+
# 创建分割窗口
|
|
214
|
+
self.splitter = wx.SplitterWindow(self)
|
|
215
|
+
|
|
216
|
+
# 创建左侧面板(包含控制区域和日志区域)
|
|
217
|
+
self.left_panel = wx.Panel(self.splitter)
|
|
218
|
+
left_sizer = wx.BoxSizer(wx.VERTICAL)
|
|
219
|
+
|
|
220
|
+
# 控制区域
|
|
221
|
+
self.control_panel = wx.Panel(self.left_panel)
|
|
222
|
+
self.init_control_panel()
|
|
223
|
+
left_sizer.Add(self.control_panel, 0, wx.EXPAND | wx.ALL, 5)
|
|
224
|
+
|
|
225
|
+
# 日志区域
|
|
226
|
+
self.log_text = wx.TextCtrl(self.left_panel, style=wx.TE_MULTILINE | wx.TE_READONLY | wx.HSCROLL)
|
|
227
|
+
self.log_text.SetBackgroundColour(wx.BLACK)
|
|
228
|
+
self.log_text.SetForegroundColour(wx.GREEN)
|
|
229
|
+
self.log_text.SetFont(wx.Font(9, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL))
|
|
230
|
+
left_sizer.Add(self.log_text, 1, wx.EXPAND | wx.ALL, 5)
|
|
231
|
+
|
|
232
|
+
self.left_panel.SetSizer(left_sizer)
|
|
233
|
+
|
|
234
|
+
# 创建设备画面
|
|
235
|
+
self.device_panel = DeviceMirrorPanel(self.splitter, device, self.log)
|
|
236
|
+
|
|
237
|
+
# 设置分割
|
|
238
|
+
self.splitter.SplitVertically(self.left_panel, self.device_panel)
|
|
239
|
+
self.splitter.SetMinimumPaneSize(200)
|
|
240
|
+
|
|
241
|
+
# 保存设备引用
|
|
242
|
+
self.device = device
|
|
243
|
+
|
|
244
|
+
def log(self, message: str):
|
|
245
|
+
"""添加日志"""
|
|
246
|
+
timestamp = time.strftime("%H:%M:%S", time.localtime())
|
|
247
|
+
wx.CallAfter(self.log_text.AppendText, f"[{timestamp}] {message}\n")
|
|
248
|
+
|
|
249
|
+
def init_control_panel(self):
|
|
250
|
+
vbox = wx.BoxSizer(wx.VERTICAL)
|
|
251
|
+
|
|
252
|
+
# 添加控制按钮
|
|
253
|
+
btn_get_resolution = wx.Button(self.control_panel, label="获取分辨率")
|
|
254
|
+
btn_get_resolution.Bind(wx.EVT_BUTTON, self.on_get_resolution)
|
|
255
|
+
vbox.Add(btn_get_resolution, 0, wx.EXPAND | wx.ALL, 5)
|
|
256
|
+
|
|
257
|
+
btn_get_orientation = wx.Button(self.control_panel, label="获取设备方向")
|
|
258
|
+
btn_get_orientation.Bind(wx.EVT_BUTTON, self.on_get_orientation)
|
|
259
|
+
vbox.Add(btn_get_orientation, 0, wx.EXPAND | wx.ALL, 5)
|
|
260
|
+
|
|
261
|
+
# 启动APP区域
|
|
262
|
+
hbox = wx.BoxSizer(wx.HORIZONTAL)
|
|
263
|
+
self.package_input = wx.TextCtrl(self.control_panel)
|
|
264
|
+
hbox.Add(self.package_input, 1, wx.EXPAND | wx.RIGHT, 5)
|
|
265
|
+
btn_launch_app = wx.Button(self.control_panel, label="启动APP")
|
|
266
|
+
btn_launch_app.Bind(wx.EVT_BUTTON, self.on_launch_app)
|
|
267
|
+
hbox.Add(btn_launch_app, 0)
|
|
268
|
+
vbox.Add(hbox, 0, wx.EXPAND | wx.ALL, 5)
|
|
269
|
+
|
|
270
|
+
btn_get_current_app = wx.Button(self.control_panel, label="获取前台APP")
|
|
271
|
+
btn_get_current_app.Bind(wx.EVT_BUTTON, self.on_get_current_app)
|
|
272
|
+
vbox.Add(btn_get_current_app, 0, wx.EXPAND | wx.ALL, 5)
|
|
273
|
+
|
|
274
|
+
self.control_panel.SetSizer(vbox)
|
|
275
|
+
|
|
276
|
+
def on_get_resolution(self, event):
|
|
277
|
+
"""获取分辨率"""
|
|
278
|
+
try:
|
|
279
|
+
width, height = self.device.screen_size
|
|
280
|
+
self.log(f"设备分辨率: {width}x{height}")
|
|
281
|
+
except Exception as e:
|
|
282
|
+
self.log(f"获取分辨率失败: {e}")
|
|
283
|
+
|
|
284
|
+
def on_get_orientation(self, event):
|
|
285
|
+
"""获取设备方向"""
|
|
286
|
+
try:
|
|
287
|
+
orientation = self.device.detect_orientation()
|
|
288
|
+
orientation_text = "横屏" if orientation == "landscape" else "竖屏"
|
|
289
|
+
self.log(f"设备方向: {orientation_text}")
|
|
290
|
+
except Exception as e:
|
|
291
|
+
self.log(f"获取设备方向失败: {e}")
|
|
292
|
+
|
|
293
|
+
def on_launch_app(self, event):
|
|
294
|
+
"""启动APP"""
|
|
295
|
+
package_name = self.package_input.GetValue().strip()
|
|
296
|
+
if not package_name:
|
|
297
|
+
self.log("请输入包名")
|
|
298
|
+
return
|
|
299
|
+
try:
|
|
300
|
+
# 使用新的 API 通过 commands 属性访问平台特定方法
|
|
301
|
+
if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'launch_app'):
|
|
302
|
+
self.device.commands.launch_app(package_name)
|
|
303
|
+
self.log(f"启动APP: {package_name}")
|
|
304
|
+
else:
|
|
305
|
+
self.log("当前设备不支持启动APP功能")
|
|
306
|
+
except Exception as e:
|
|
307
|
+
self.log(f"启动APP失败: {e}")
|
|
308
|
+
|
|
309
|
+
def on_get_current_app(self, event):
|
|
310
|
+
"""获取前台APP"""
|
|
311
|
+
try:
|
|
312
|
+
# 使用新的 API 通过 commands 属性访问平台特定方法
|
|
313
|
+
if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'current_package'):
|
|
314
|
+
package = self.device.commands.current_package()
|
|
315
|
+
if package:
|
|
316
|
+
self.log(f"前台APP: {package}")
|
|
317
|
+
else:
|
|
318
|
+
self.log("未获取到前台APP")
|
|
319
|
+
else:
|
|
320
|
+
self.log("当前设备不支持获取前台APP功能")
|
|
321
|
+
except Exception as e:
|
|
322
|
+
self.log(f"获取前台APP失败: {e}")
|
|
323
|
+
|
|
324
|
+
def on_quit(self, event):
|
|
325
|
+
self.device_panel.is_running = False
|
|
326
|
+
self.Close()
|
|
327
|
+
|
|
328
|
+
def show_device_mirror(device: Device):
|
|
329
|
+
"""显示设备镜像窗口"""
|
|
330
|
+
app = wx.App()
|
|
331
|
+
frame = DeviceMirrorFrame(device)
|
|
332
|
+
frame.Show()
|
|
333
|
+
app.MainLoop()
|
|
334
|
+
|
|
335
|
+
if __name__ == "__main__":
|
|
336
|
+
# 测试代码
|
|
337
|
+
from kotonebot.client.device import AndroidDevice
|
|
338
|
+
from kotonebot.client.implements.adb import AdbImpl
|
|
339
|
+
from kotonebot.client.implements.uiautomator2 import UiAutomator2Impl
|
|
340
|
+
from adbutils import adb
|
|
341
|
+
|
|
342
|
+
print("server version:", adb.server_version())
|
|
343
|
+
adb.connect("127.0.0.1:5555")
|
|
344
|
+
print("devices:", adb.device_list())
|
|
345
|
+
d = adb.device_list()[-1]
|
|
346
|
+
|
|
347
|
+
# 使用新的 API
|
|
348
|
+
dd = AndroidDevice(d)
|
|
349
|
+
adb_imp = AdbImpl(d) # 直接传入 adb 连接
|
|
350
|
+
dd._touch = adb_imp
|
|
351
|
+
dd._screenshot = UiAutomator2Impl(dd) # UiAutomator2Impl 可能还需要 device 对象
|
|
352
|
+
dd.commands = adb_imp # 设置 Android 特定命令
|
|
353
|
+
|
|
354
|
+
show_device_mirror(dd)
|