kotonebot 0.1.0__py3-none-any.whl

kotonebot/primitives/geometry.py

@@ -0,0 +1,290 @@
+from typing import Generic, TypeVar, TypeGuard, overload
+
+T = TypeVar('T')
+
+class Vector2D(Generic[T]):
+    """2D 坐标类"""
+    def __init__(self, x: T, y: T, *, name: str | None = None):
+        self.x = x
+        self.y = y
+        self.name: str | None = name
+        """坐标的名称。"""
+
+    def __getitem__(self, item: int):
+        if item == 0:
+            return self.x
+        elif item == 1:
+            return self.y
+        else:
+            raise IndexError
+
+    def __repr__(self) -> str:
+        return f'Point<"{self.name}" at ({self.x}, {self.y})>'
+
+    def __str__(self) -> str:
+        return f'({self.x}, {self.y})'
+
+
+class Vector3D(Generic[T]):
+    """三元组类。"""
+    def __init__(self, x: T, y: T, z: T, *, name: str | None = None):
+        self.x = x
+        self.y = y
+        self.z = z
+        self.name: str | None = name
+        """坐标的名称。"""
+
+    def __getitem__(self, item: int):
+        if item == 0:
+            return self.x
+        elif item == 1:
+            return self.y
+        elif item == 2:
+            return self.z
+        else:
+            raise IndexError
+
+    @property
+    def xyz(self) -> tuple[T, T, T]:
+        """
+        三元组 (x, y, z)。OpenCV 格式的坐标。
+        """
+        return self.x, self.y, self.z
+
+    @property
+    def xy(self) -> tuple[T, T]:
+        """
+        二元组 (x, y)。OpenCV 格式的坐标。
+        """
+        return self.x, self.y
+
+class Vector4D(Generic[T]):
+    """四元组类。"""
+    def __init__(self, x: T, y: T, z: T, w: T, *, name: str | None = None):
+        self.x = x
+        self.y = y
+        self.z = z
+        self.w = w
+        self.name: str | None = name
+        """坐标的名称。"""
+
+    def __getitem__(self, item: int):
+        if item == 0:
+            return self.x
+        elif item == 1:
+            return self.y
+        elif item == 2:
+            return self.z
+        elif item == 3:
+            return self.w
+        else:
+            raise IndexError
+
+Size = Vector2D[int]
+"""尺寸。相当于 Vector2D[int]"""
+RectTuple = tuple[int, int, int, int]
+"""矩形。(x, y, w, h)"""
+PointTuple = tuple[int, int]
+"""点。(x, y)"""
+
+class Point(Vector2D[int]):
+    """点。"""
+    
+    @property
+    def xy(self) -> PointTuple:
+        """
+        二元组 (x, y)。OpenCV 格式的坐标。
+        """
+        return self.x, self.y
+    
+    def offset(self, dx: int, dy: int) -> 'Point':
+        """
+        偏移坐标。
+        
+        :param dx: 偏移量。
+        :param dy: 偏移量。
+        :return: 偏移后的坐标。
+        """
+        return Point(self.x + dx, self.y + dy, name=self.name)
+    
+    def __add__(self, other: 'Point | PointTuple') -> 'Point':
+        """
+        相加。
+        
+        :param other: 另一个 Point 对象或二元组 (x: int, y: int)。
+        :return: 相加后的点。
+        """
+        if isinstance(other, Point):
+            return Point(self.x + other.x, self.y + other.y, name=self.name)
+        else:
+            return Point(self.x + other[0], self.y + other[1], name=self.name)
+        
+    def __sub__(self, other: 'Point | PointTuple') -> 'Point':
+        """
+        相减。
+        
+        :param other: 另一个 Point 对象或二元组 (x: int, y: int)。
+        :return: 相减后的点。
+        """
+        if isinstance(other, Point):
+            return Point(self.x - other.x, self.y - other.y, name=self.name)
+        else:
+            return Point(self.x - other[0], self.y - other[1], name=self.name)
+
+class Rect:
+    """
+    矩形类。
+    """
+    def __init__(
+        self,
+        x: int | None = None,
+        y: int | None = None,
+        w: int | None = None,
+        h: int | None = None,
+        *,
+        xywh: RectTuple | None = None,
+        name: str | None = None,
+    ):
+        """
+        从给定的坐标信息创建矩形。
+        
+        参数 `x`, `y`, `w`, `h` 和 `xywh` 必须至少指定一组。
+        
+        :param x: 矩形左上角的 X 坐标。
+        :param y: 矩形左上角的 Y 坐标。
+        :param w: 矩形的宽度。
+        :param h: 矩形的高度。
+        :param xywh: 四元组 (x, y, w, h)。
+        :param name: 矩形的名称。
+        :raises ValueError: 提供的坐标参数不完整时抛出。
+        """
+        if xywh is not None:
+            x, y, w, h = xywh
+        elif (
+            x is not None and
+            y is not None and
+            w is not None and
+            h is not None
+        ):
+            pass
+        else:
+            raise ValueError('Either xywh or x, y, w, h must be provided.')
+        
+        self.x1 = x
+        """矩形左上角的 X 坐标。"""
+        self.y1 = y
+        """矩形左上角的 Y 坐标。"""
+        self.w = w
+        """矩形的宽度。"""
+        self.h = h
+        """矩形的高度。"""
+        self.name: str | None = name
+        """矩形的名称。"""
+
+    @classmethod
+    def from_xyxy(cls, x1: int, y1: int, x2: int, y2: int) -> 'Rect':
+        """
+        从 (x1, y1, x2, y2) 创建矩形。
+        :return: 创建结果。
+        """
+        return cls(x1, y1, x2 - x1, y2 - y1)
+
+    @property
+    def x2(self) -> int:
+        """矩形右下角的 X 坐标。"""
+        return self.x1 + self.w
+
+    @x2.setter
+    def x2(self, value: int):
+        self.w = value - self.x1
+
+    @property
+    def y2(self) -> int:
+        """矩形右下角的 Y 坐标。"""
+        return self.y1 + self.h
+
+    @y2.setter
+    def y2(self, value: int):
+        self.h = value - self.y1
+
+    @property
+    def xywh(self) -> RectTuple:
+        """
+        四元组 (x1, y1, w, h)。OpenCV 格式的坐标。
+        """
+        return self.x1, self.y1, self.w, self.h
+
+    @property
+    def xyxy(self) -> RectTuple:
+        """
+        四元组 (x1, y1, x2, y2)。
+        """
+        return self.x1, self.y1, self.x2, self.y2
+
+    @property
+    def top_left(self) -> Point:
+        """
+        矩形的左上角点。
+        """
+        if self.name:
+            name = "Left-top of rect "+ self.name
+        else:
+            name = None
+        return Point(self.x1, self.y1, name=name)
+    
+    @property
+    def bottom_right(self) -> Point:
+        """
+        矩形的右下角点。
+        """
+        if self.name:
+            name = "Right-bottom of rect "+ self.name
+        else:
+            name = None
+        return Point(self.x2, self.y2, name=name)
+    
+    @property
+    def left_bottom(self) -> Point:
+        """
+        矩形的左下角点。
+        """
+        if self.name:
+            name = "Left-bottom of rect "+ self.name
+        else:
+            name = None
+        return Point(self.x1, self.y2, name=name)
+    
+    @property
+    def right_top(self) -> Point:
+        """
+        矩形的右上角点。
+        """
+        if self.name:
+            name = "Right-top of rect "+ self.name
+        else:
+            name = None
+        return Point(self.x2, self.y1, name=name)
+    
+    @property
+    def center(self) -> Point:
+        """
+        矩形的中心点。
+        """
+        if self.name:
+            name = "Center of rect "+ self.name
+        else:
+            name = None
+        return Point(self.x1 + self.w // 2, self.y1 + self.h // 2, name=name)
+
+    def __repr__(self) -> str:
+        return f'Rect<"{self.name}" at (x={self.x1}, y={self.y1}, w={self.w}, h={self.h})>'
+
+    def __str__(self) -> str:
+        return f'(x={self.x1}, y={self.y1}, w={self.w}, h={self.h})'
+
+
+def is_point(obj: object) -> TypeGuard[Point]:
+    return isinstance(obj, Point)
+
+def is_rect(obj: object) -> TypeGuard[Rect]:
+    return isinstance(obj, Rect)

kotonebot/primitives/visual.py

@@ -0,0 +1,63 @@
+import logging
+
+from cv2.typing import MatLike
+
+from .geometry import Size
+from kotonebot.util import cv2_imread
+
+logger = logging.getLogger(__name__)
+
+class Image:
+    """
+    图像类。
+    """
+    def __init__(
+        self,
+        pixels: MatLike | None = None,
+        file_path: str | None = None,
+        lazy_load: bool = False,
+        name: str | None = None,
+        description: str | None = None
+    ):
+        """
+        从内存数据或图像文件创建图像类。
+        
+        :param pixels: 图像数据。格式必须为 BGR。
+        :param file_path: 图像文件路径。
+        :param lazy_load: 是否延迟加载图像数据。
+            若为 False，立即载入，否则仅当访问图像数据时才载入。仅当从文件创建图像类时生效。
+        :param name: 图像名称。
+        :param description: 图像描述。
+        """
+        self.name: str | None = name
+        """图像名称。"""
+        self.description: str | None = description
+        """图像描述。"""
+        self.file_path: str | None = file_path
+        """图像的文件路径。"""
+        self.__pixels: MatLike | None = None
+        # 立即加载
+        if not lazy_load and self.file_path:
+            _ = self.pixels
+        # 传入像素数据而不是文件
+        if pixels is not None:
+            self.__pixels = pixels
+
+    @property
+    def pixels(self) -> MatLike:
+        """图像的像素数据。"""
+        if self.__pixels is None:
+            if not self.file_path:
+                raise ValueError('Either pixels or file_path must be provided.')
+            logger.debug('Loading image "%s" from %s...', self.name or '(unnamed)', self.file_path)
+            self.__pixels = cv2_imread(self.file_path)
+        return self.__pixels
+
+    @property
+    def size(self) -> Size:
+        return Size(self.pixels.shape[1], self.pixels.shape[0])
+
+class Template(Image):
+    """
+    模板图像类。
+    """

kotonebot/tools/mirror.py

@@ -0,0 +1,354 @@
+import wx
+import cv2
+import numpy as np
+import time
+from typing import Optional, Tuple, Callable
+from threading import Thread, Lock
+from cv2.typing import MatLike
+from queue import Queue
+
+from kotonebot.client.device import Device
+
+class DeviceMirrorPanel(wx.Panel):
+    def __init__(self, parent, device: Device, log_callback=None):
+        super().__init__(parent)
+        self.device = device
+        self.screen_bitmap: Optional[wx.Bitmap] = None
+        self.fps = 0
+        self.last_frame_time = time.time()
+        self.frame_count = 0
+        self.is_running = True
+        self.lock = Lock()
+        self.last_mouse_pos = (0, 0)
+        self.is_dragging = False
+        self.screenshot_interval = 0  # 截图耗时(ms)
+        self.log_callback = log_callback
+        self.operation_queue = Queue()
+        
+        # 设置背景色为黑色
+        self.SetBackgroundColour(wx.BLACK)
+        
+        # 双缓冲，减少闪烁
+        self.SetDoubleBuffered(True)
+        
+        # 绑定事件
+        self.Bind(wx.EVT_PAINT, self.on_paint)
+        self.Bind(wx.EVT_SIZE, self.on_size)
+        self.Bind(wx.EVT_LEFT_DOWN, self.on_left_down)
+        self.Bind(wx.EVT_LEFT_UP, self.on_left_up)
+        self.Bind(wx.EVT_MOTION, self.on_motion)
+        
+        # 启动刷新线程
+        self.update_thread = Thread(target=self.update_screen, daemon=True)
+        self.update_thread.start()
+        
+        # 启动操作处理线程
+        self.operation_thread = Thread(target=self.process_operations, daemon=True)
+        self.operation_thread.start()
+        
+    def process_operations(self):
+        """处理设备操作的线程"""
+        while self.is_running:
+            try:
+                operation = self.operation_queue.get()
+                if operation is not None:
+                    operation()
+                self.operation_queue.task_done()
+            except Exception as e:
+                if self.log_callback:
+                    self.log_callback(f"操作执行错误: {e}")
+                    
+    def execute_device_operation(self, operation: Callable):
+        """将设备操作添加到队列"""
+        self.operation_queue.put(operation)
+
+    def update_screen(self):
+        while self.is_running:
+            try:
+                # 获取设备截图并计时
+                start_time = time.time()
+                frame = self.device.screenshot()
+                end_time = time.time()
+                self.screenshot_interval = int((end_time - start_time) * 1000)
+                
+                if frame is None:
+                    continue
+                    
+                # 计算FPS
+                current_time = time.time()
+                self.frame_count += 1
+                if current_time - self.last_frame_time >= 1.0:
+                    self.fps = self.frame_count
+                    self.frame_count = 0
+                    self.last_frame_time = current_time
+                
+                # 转换为wx.Bitmap
+                height, width = frame.shape[:2]
+                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                wximage = wx.Bitmap.FromBuffer(width, height, frame)
+                
+                with self.lock:
+                    self.screen_bitmap = wximage
+                
+                # 请求重绘
+                wx.CallAfter(self.Refresh)
+                
+                # 控制刷新率
+                time.sleep(1/60)
+                
+            except Exception as e:
+                print(f"Error updating screen: {e}")
+                time.sleep(1)
+    
+    def on_paint(self, event):
+        dc = wx.BufferedPaintDC(self)
+        
+        # 清空背景
+        dc.SetBackground(wx.Brush(wx.BLACK))
+        dc.Clear()
+        
+        if not self.screen_bitmap:
+            return
+            
+        # 绘制设备画面
+        with self.lock:
+            # 计算缩放比例，保持宽高比
+            panel_width, panel_height = self.GetSize()
+            bitmap_width = self.screen_bitmap.GetWidth()
+            bitmap_height = self.screen_bitmap.GetHeight()
+            
+            scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
+            scaled_width = int(bitmap_width * scale)
+            scaled_height = int(bitmap_height * scale)
+            
+            # 居中显示
+            x = (panel_width - scaled_width) // 2
+            y = (panel_height - scaled_height) // 2
+            
+            if scale != 1:
+                img = self.screen_bitmap.ConvertToImage()
+                img = img.Scale(scaled_width, scaled_height, wx.IMAGE_QUALITY_HIGH)
+                bitmap = wx.Bitmap(img)
+            else:
+                bitmap = self.screen_bitmap
+                
+            dc.DrawBitmap(bitmap, x, y)
+        
+        # 绘制FPS和截图时间
+        dc.SetTextForeground(wx.GREEN)
+        dc.SetFont(wx.Font(10, wx.FONTFAMILY_DEFAULT, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_BOLD))
+        dc.DrawText(f"FPS: {self.fps}", 10, 10)
+        dc.DrawText(f"Interval: {self.screenshot_interval}ms", 10, 30)
+    
+    def on_size(self, event):
+        self.Refresh()
+        event.Skip()
+        
+    def get_device_coordinates(self, x: int, y: int) -> Tuple[int, int]:
+        """将面板坐标转换为设备坐标"""
+        if not self.screen_bitmap:
+            return (0, 0)
+            
+        panel_width, panel_height = self.GetSize()
+        bitmap_width = self.screen_bitmap.GetWidth()
+        bitmap_height = self.screen_bitmap.GetHeight()
+        
+        scale = min(panel_width/bitmap_width, panel_height/bitmap_height)
+        scaled_width = int(bitmap_width * scale)
+        scaled_height = int(bitmap_height * scale)
+        
+        # 计算显示区域的偏移
+        x_offset = (panel_width - scaled_width) // 2
+        y_offset = (panel_height - scaled_height) // 2
+        
+        # 转换坐标
+        device_x = int((x - x_offset) / scale)
+        device_y = int((y - y_offset) / scale)
+        
+        # 确保坐标在设备范围内
+        device_x = max(0, min(device_x, bitmap_width-1))
+        device_y = max(0, min(device_y, bitmap_height-1))
+        
+        return (device_x, device_y)
+    
+    def on_left_down(self, event):
+        self.last_mouse_pos = event.GetPosition()
+        self.is_dragging = True
+        event.Skip()
+        
+    def on_left_up(self, event):
+        if not self.is_dragging:
+            return
+            
+        self.is_dragging = False
+        pos = event.GetPosition()
+        
+        # 如果鼠标位置没有明显变化，执行点击
+        if abs(pos[0] - self.last_mouse_pos[0]) < 5 and abs(pos[1] - self.last_mouse_pos[1]) < 5:
+            device_x, device_y = self.get_device_coordinates(*pos)
+            self.execute_device_operation(lambda: self.device.click(device_x, device_y))
+            if self.log_callback:
+                self.log_callback(f"点击: ({device_x}, {device_y})")
+        else:
+            # 执行滑动
+            start_x, start_y = self.get_device_coordinates(*self.last_mouse_pos)
+            end_x, end_y = self.get_device_coordinates(*pos)
+            self.execute_device_operation(lambda: self.device.swipe(start_x, start_y, end_x, end_y))
+            if self.log_callback:
+                self.log_callback(f"滑动: ({start_x}, {start_y}) -> ({end_x}, {end_y})")
+            
+        event.Skip()
+        
+    def on_motion(self, event):
+        if not self.is_dragging:
+            event.Skip()
+            return
+            
+        event.Skip()
+
+class DeviceMirrorFrame(wx.Frame):
+    def __init__(self, device: Device):
+        super().__init__(None, title="设备镜像", size=(800, 600))
+        
+        # 创建分割窗口
+        self.splitter = wx.SplitterWindow(self)
+        
+        # 创建左侧面板（包含控制区域和日志区域）
+        self.left_panel = wx.Panel(self.splitter)
+        left_sizer = wx.BoxSizer(wx.VERTICAL)
+        
+        # 控制区域
+        self.control_panel = wx.Panel(self.left_panel)
+        self.init_control_panel()
+        left_sizer.Add(self.control_panel, 0, wx.EXPAND | wx.ALL, 5)
+        
+        # 日志区域
+        self.log_text = wx.TextCtrl(self.left_panel, style=wx.TE_MULTILINE | wx.TE_READONLY | wx.HSCROLL)
+        self.log_text.SetBackgroundColour(wx.BLACK)
+        self.log_text.SetForegroundColour(wx.GREEN)
+        self.log_text.SetFont(wx.Font(9, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL))
+        left_sizer.Add(self.log_text, 1, wx.EXPAND | wx.ALL, 5)
+        
+        self.left_panel.SetSizer(left_sizer)
+        
+        # 创建设备画面
+        self.device_panel = DeviceMirrorPanel(self.splitter, device, self.log)
+        
+        # 设置分割
+        self.splitter.SplitVertically(self.left_panel, self.device_panel)
+        self.splitter.SetMinimumPaneSize(200)
+        
+        # 保存设备引用
+        self.device = device
+        
+    def log(self, message: str):
+        """添加日志"""
+        timestamp = time.strftime("%H:%M:%S", time.localtime())
+        wx.CallAfter(self.log_text.AppendText, f"[{timestamp}] {message}\n")
+        
+    def init_control_panel(self):
+        vbox = wx.BoxSizer(wx.VERTICAL)
+        
+        # 添加控制按钮
+        btn_get_resolution = wx.Button(self.control_panel, label="获取分辨率")
+        btn_get_resolution.Bind(wx.EVT_BUTTON, self.on_get_resolution)
+        vbox.Add(btn_get_resolution, 0, wx.EXPAND | wx.ALL, 5)
+        
+        btn_get_orientation = wx.Button(self.control_panel, label="获取设备方向")
+        btn_get_orientation.Bind(wx.EVT_BUTTON, self.on_get_orientation)
+        vbox.Add(btn_get_orientation, 0, wx.EXPAND | wx.ALL, 5)
+        
+        # 启动APP区域
+        hbox = wx.BoxSizer(wx.HORIZONTAL)
+        self.package_input = wx.TextCtrl(self.control_panel)
+        hbox.Add(self.package_input, 1, wx.EXPAND | wx.RIGHT, 5)
+        btn_launch_app = wx.Button(self.control_panel, label="启动APP")
+        btn_launch_app.Bind(wx.EVT_BUTTON, self.on_launch_app)
+        hbox.Add(btn_launch_app, 0)
+        vbox.Add(hbox, 0, wx.EXPAND | wx.ALL, 5)
+        
+        btn_get_current_app = wx.Button(self.control_panel, label="获取前台APP")
+        btn_get_current_app.Bind(wx.EVT_BUTTON, self.on_get_current_app)
+        vbox.Add(btn_get_current_app, 0, wx.EXPAND | wx.ALL, 5)
+        
+        self.control_panel.SetSizer(vbox)
+        
+    def on_get_resolution(self, event):
+        """获取分辨率"""
+        try:
+            width, height = self.device.screen_size
+            self.log(f"设备分辨率: {width}x{height}")
+        except Exception as e:
+            self.log(f"获取分辨率失败: {e}")
+            
+    def on_get_orientation(self, event):
+        """获取设备方向"""
+        try:
+            orientation = self.device.detect_orientation()
+            orientation_text = "横屏" if orientation == "landscape" else "竖屏"
+            self.log(f"设备方向: {orientation_text}")
+        except Exception as e:
+            self.log(f"获取设备方向失败: {e}")
+            
+    def on_launch_app(self, event):
+        """启动APP"""
+        package_name = self.package_input.GetValue().strip()
+        if not package_name:
+            self.log("请输入包名")
+            return
+        try:
+            # 使用新的 API 通过 commands 属性访问平台特定方法
+            if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'launch_app'):
+                self.device.commands.launch_app(package_name)
+                self.log(f"启动APP: {package_name}")
+            else:
+                self.log("当前设备不支持启动APP功能")
+        except Exception as e:
+            self.log(f"启动APP失败: {e}")
+            
+    def on_get_current_app(self, event):
+        """获取前台APP"""
+        try:
+            # 使用新的 API 通过 commands 属性访问平台特定方法
+            if hasattr(self.device, 'commands') and hasattr(self.device.commands, 'current_package'):
+                package = self.device.commands.current_package()
+                if package:
+                    self.log(f"前台APP: {package}")
+                else:
+                    self.log("未获取到前台APP")
+            else:
+                self.log("当前设备不支持获取前台APP功能")
+        except Exception as e:
+            self.log(f"获取前台APP失败: {e}")
+            
+    def on_quit(self, event):
+        self.device_panel.is_running = False
+        self.Close()
+
+def show_device_mirror(device: Device):
+    """显示设备镜像窗口"""
+    app = wx.App()
+    frame = DeviceMirrorFrame(device)
+    frame.Show()
+    app.MainLoop()
+
+if __name__ == "__main__":
+    # 测试代码
+    from kotonebot.client.device import AndroidDevice
+    from kotonebot.client.implements.adb import AdbImpl
+    from kotonebot.client.implements.uiautomator2 import UiAutomator2Impl
+    from adbutils import adb
+
+    print("server version:", adb.server_version())
+    adb.connect("127.0.0.1:5555")
+    print("devices:", adb.device_list())
+    d = adb.device_list()[-1]
+
+    # 使用新的 API
+    dd = AndroidDevice(d)
+    adb_imp = AdbImpl(d)  # 直接传入 adb 连接
+    dd._touch = adb_imp
+    dd._screenshot = UiAutomator2Impl(dd)  # UiAutomator2Impl 可能还需要 device 对象
+    dd.commands = adb_imp  # 设置 Android 特定命令
+
+    show_device_mirror(dd)