kotonebot 0.1.0__py3-none-any.whl

kotonebot/client/host/windows_common.py

@@ -0,0 +1,55 @@
+from abc import ABC
+from typing import Literal
+from typing_extensions import assert_never
+
+from kotonebot import logging
+from kotonebot.client.device import WindowsDevice
+from .protocol import Device, WindowsHostConfig, RemoteWindowsHostConfig
+
+logger = logging.getLogger(__name__)
+WindowsRecipes = Literal['windows', 'remote_windows']
+
+# Windows 相关的配置类型联合
+WindowsHostConfigs = WindowsHostConfig | RemoteWindowsHostConfig
+
+class CommonWindowsCreateDeviceMixin(ABC):
+    """
+    通用 Windows 创建设备的 Mixin。
+    该 Mixin 定义了创建 Windows 设备的通用接口。
+    """
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+    
+    def create_device(self, recipe: WindowsRecipes, config: WindowsHostConfigs) -> Device:
+        """
+        创建 Windows 设备。
+        """
+        match recipe:
+            case 'windows':
+                if not isinstance(config, WindowsHostConfig):
+                    raise ValueError(f"Expected WindowsHostConfig for 'windows' recipe, got {type(config)}")
+                from kotonebot.client.implements.windows import WindowsImpl
+                d = WindowsDevice()
+                impl = WindowsImpl(
+                    device=d,
+                    window_title=config.window_title,
+                    ahk_exe_path=config.ahk_exe_path
+                )
+                d._screenshot = impl
+                d._touch = impl
+                return d
+            case 'remote_windows':
+                if not isinstance(config, RemoteWindowsHostConfig):
+                    raise ValueError(f"Expected RemoteWindowsHostConfig for 'remote_windows' recipe, got {type(config)}")
+                from kotonebot.client.implements.remote_windows import RemoteWindowsImpl
+                d = WindowsDevice()
+                impl = RemoteWindowsImpl(
+                    device=d,
+                    host=config.host,
+                    port=config.port
+                )
+                d._screenshot = impl
+                d._touch = impl
+                return d
+            case _:
+                assert_never(f'Unsupported Windows recipe: {recipe}')

kotonebot/client/implements/__init__.py

@@ -0,0 +1,7 @@
+# 导入所有内置实现，以触发它们的 @register_impl 装饰器
+from . import adb  # noqa: F401
+from . import adb_raw  # noqa: F401
+from . import remote_windows  # noqa: F401
+from . import uiautomator2  # noqa: F401
+from . import windows  # noqa: F401
+from . import nemu_ipc  # noqa: F401

kotonebot/client/implements/adb.py

@@ -0,0 +1,85 @@
+import logging
+from typing import cast
+from typing_extensions import override
+
+import cv2
+import numpy as np
+from cv2.typing import MatLike
+from adbutils._device import AdbDevice as AdbUtilsDevice
+
+from ..device import AndroidDevice
+from ..protocol import AndroidCommandable, Touchable, Screenshotable
+from ..registration import ImplConfig
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+# 定义配置模型
+@dataclass
+class AdbImplConfig(ImplConfig):
+    addr: str
+    connect: bool = True
+    disconnect: bool = True
+    device_serial: str | None = None
+    timeout: float = 180
+
+class AdbImpl(AndroidCommandable, Touchable, Screenshotable):
+    def __init__(self, adb_connection: AdbUtilsDevice):
+        self.adb = adb_connection
+
+    @override
+    def launch_app(self, package_name: str) -> None:
+        self.adb.shell(f"monkey -p {package_name} 1")
+
+    @override
+    def current_package(self) -> str | None:
+        # https://blog.csdn.net/guangdeshishe/article/details/117154406
+        result_text = self.adb.shell('dumpsys activity top | grep ACTIVITY | tail -n 1')
+        logger.debug(f"adb returned: {result_text}")
+        if not isinstance(result_text, str):
+            logger.error(f"Invalid result_text: {result_text}")
+            return None
+        result_text = result_text.strip()
+        if result_text == '':
+            logger.error("No current package found")
+            return None
+        _, activity, *_ = result_text.split(' ')
+        package = activity.split('/')[0]
+        return package
+
+    def adb_shell(self, cmd: str) -> str:
+        """执行 ADB shell 命令"""
+        return cast(str, self.adb.shell(cmd))
+
+    @override
+    def detect_orientation(self):
+        # 判断方向：https://stackoverflow.com/questions/10040624/check-if-device-is-landscape-via-adb
+        # 但是上面这种方法不准确
+        # 因此这里直接通过截图判断方向
+        img = self.screenshot()
+        if img.shape[0] > img.shape[1]:
+            return 'portrait'
+        return 'landscape'
+    
+    @property
+    def screen_size(self) -> tuple[int, int]:
+        ret = cast(str, self.adb.shell("wm size")).strip('Physical size: ')
+        spiltted = tuple(map(int, ret.split("x")))
+        # 检测当前方向
+        orientation = self.detect_orientation()
+        landscape = orientation == 'landscape'
+        spiltted = tuple(sorted(spiltted, reverse=landscape))
+        if len(spiltted) != 2:
+            raise ValueError(f"Invalid screen size: {ret}")
+        return spiltted
+
+    def screenshot(self) -> MatLike:
+        return cv2.cvtColor(np.array(self.adb.screenshot()), cv2.COLOR_RGB2BGR)
+
+    def click(self, x: int, y: int) -> None:
+        self.adb.shell(f"input tap {x} {y}")
+
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, duration: float | None = None) -> None:
+        if duration is not None:
+            logger.warning("Swipe duration is not supported with AdbDevice. Ignoring duration.")
+        self.adb.shell(f"input touchscreen swipe {x1} {y1} {x2} {y2}")

kotonebot/client/implements/adb_raw.py

@@ -0,0 +1,159 @@
+import os
+import time
+import subprocess
+import struct
+from threading import Thread, Lock
+from functools import cached_property
+from typing_extensions import override
+
+import cv2
+import numpy as np
+from cv2.typing import MatLike
+from adbutils._utils import adb_path
+
+from .adb import AdbImpl
+from adbutils._device import AdbDevice as AdbUtilsDevice
+from kotonebot import logging
+
+logger = logging.getLogger(__name__)
+
+WAIT_TIMEOUT = 10
+MAX_RETRY_COUNT = 5
+SCRIPT: str = """#!/bin/sh
+while true; do
+    screencap
+    sleep 0.3
+done
+"""
+
+class AdbRawImpl(AdbImpl):
+    def __init__(self, adb_connection: AdbUtilsDevice):
+        super().__init__(adb_connection)
+        self.__worker: Thread | None = None
+        self.__process: subprocess.Popen | None = None
+        self.__data: MatLike | None = None
+        self.__retry_count = 0
+        self.__lock = Lock()
+        self.__stopping = False
+
+    def __cleanup_worker(self) -> None:
+        if self.__process:
+            try:
+                self.__process.kill()
+            except:
+                pass
+            self.__process = None
+        if self.__worker:
+            try:
+                self.__worker.join()
+            except:
+                pass
+            self.__worker = None
+        self.__data = None
+
+    def __start_worker(self) -> None:
+        self.__stopping = True
+        self.__cleanup_worker()
+        self.__stopping = False
+        self.__worker = Thread(target=self.__worker_thread_with_retry, daemon=True)
+        self.__worker.start()
+
+    def __worker_thread_with_retry(self) -> None:
+        try:
+            self.__worker_thread()
+        except Exception as e:
+            logger.error(f"Worker thread failed: {e}")
+            with self.__lock:
+                self.__retry_count += 1
+            raise
+
+    def __worker_thread(self) -> None:
+        with open('screenshot.sh', 'w', encoding='utf-8', newline='\n') as f:
+            f.write(SCRIPT)
+        self.adb.push('screenshot.sh', '/data/local/tmp/screenshot.sh')
+        self.adb.shell(f'chmod 755 /data/local/tmp/screenshot.sh')
+        os.remove('screenshot.sh')
+
+        cmd = fr'{adb_path()} -s {self.adb.serial} exec-out "sh /data/local/tmp/screenshot.sh"'
+        self.__process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+        
+        while not self.__stopping and self.__process.poll() is None:
+            if self.__process.stdout is None:
+                logger.error("Failed to get stdout from process")
+                continue
+            
+            # 解析 header  
+            # https://stackoverflow.com/questions/22034959/what-format-does-adb-screencap-sdcard-screenshot-raw-produce-without-p-f
+            if self.__api_level >= 26:
+                metadata = self.__process.stdout.read(16)
+                w, h, p, c = struct.unpack('<IIII', metadata)
+                # w=width, h=height, p=pixel_format, c=color_space
+                # 详见：https://android.googlesource.com/platform/frameworks/base/+/26a2b97dbe48ee45e9ae70110714048f2f360f97/cmds/screencap/screencap.cpp#209
+            else:
+                metadata = self.__process.stdout.read(12)
+                w, h, p = struct.unpack('<III', metadata)
+            if p == 1: # PixelFormat.RGBA_8888
+                channel = 4
+            else:
+                raise ValueError(f"Unsupported pixel format: {p}")
+            data_size = w * h * channel
+
+            if (data_size < 100 * 100 * 4) or (data_size > 3000 * 3000 * 4):
+                raise ValueError(f"Invaild data_size: {w}x{h}.")
+
+            # 读取图像数据
+            # logger.verbose(f"receiving image data: {w}x{h} {data_size} bytes")
+            image_data = self.__process.stdout.read(data_size)
+            if not isinstance(image_data, bytes) or len(image_data) != data_size:
+                logger.error(f"Failed to read image data, expected {data_size} bytes but got {len(image_data) if isinstance(image_data, bytes) else 'non-bytes'}")
+                raise RuntimeError("Failed to read image data")
+                
+            np_data = np.frombuffer(image_data, np.uint8)
+            np_data = np_data.reshape(h, w, channel)
+            self.__data = cv2.cvtColor(np_data, cv2.COLOR_RGBA2BGR)
+
+    @cached_property
+    def __api_level(self) -> int:
+        try:
+            output = self.adb.shell("getprop ro.build.version.sdk")
+            assert isinstance(output, str)
+            return int(output.strip())
+        except Exception as e:
+            logger.error(f"Failed to get API level: {e}")
+            return 0
+
+    @override
+    def screenshot(self) -> MatLike:
+        with self.__lock:
+            if self.__retry_count >= MAX_RETRY_COUNT:
+                raise RuntimeError(f"Maximum retry count ({MAX_RETRY_COUNT}) exceeded")
+
+            if not self.__worker or (self.__worker and not self.__worker.is_alive()):
+                self.__start_worker()
+        
+        start_time = time.time()
+        while self.__data is None:
+            time.sleep(0.01)
+            if time.time() - start_time > WAIT_TIMEOUT:
+                logger.warning("Screenshot timeout, cleaning up and restarting worker...")
+                with self.__lock:
+                    if self.__retry_count < MAX_RETRY_COUNT:
+                        self.__start_worker()
+                        start_time = time.time()  # 重置超时计时器
+                        continue
+                    else:
+                        raise RuntimeError(f"Maximum retry count ({MAX_RETRY_COUNT}) exceeded")
+            
+            # 检查 worker 是否还活着
+            if self.__worker and not self.__worker.is_alive():
+                with self.__lock:
+                    if self.__retry_count < MAX_RETRY_COUNT:
+                        logger.warning("Worker thread died, restarting...")
+                        self.__start_worker()
+                    else:
+                        raise RuntimeError(f"Maximum retry count ({MAX_RETRY_COUNT}) exceeded")
+
+        logger.verbose(f"adb raw screenshot wait time: {time.time() - start_time:.4f}s")
+        data = self.__data
+        self.__data = None
+        return data

kotonebot/client/implements/nemu_ipc/__init__.py

@@ -0,0 +1,8 @@
+from .external_renderer_ipc import ExternalRendererIpc
+from .nemu_ipc import NemuIpcImpl, NemuIpcImplConfig
+
+__all__ = [
+    "ExternalRendererIpc",
+    "NemuIpcImpl",
+    "NemuIpcImplConfig",
+]

kotonebot/client/implements/nemu_ipc/external_renderer_ipc.py

@@ -0,0 +1,280 @@
+import ctypes
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class NemuIpcIncompatible(RuntimeError):
+    """MuMu12 IPC 环境不兼容或 DLL 加载失败"""
+
+
+class ExternalRendererIpc:
+    r"""对 `external_renderer_ipc.dll` 的轻量封装。
+
+    该类仅处理 DLL 加载与原型声明，并提供带有类型提示的薄包装方法，
+    方便在其他模块中调用且保持类型安全。
+    传入参数为 MuMu 根目录（如 F:\Apps\Netease\MuMuPlayer-12.0）。
+    """
+
+    def __init__(self, mumu_root_folder: str):
+        if os.name != "nt":
+            raise NemuIpcIncompatible("ExternalRendererIpc only supports Windows.")
+
+        self.lib = self.__load_dll(mumu_root_folder)
+        self.raise_on_error: bool = True
+        """是否在调用 DLL 函数失败时抛出异常。"""
+        self.__declare_prototypes()
+
+    def connect(self, nemu_folder: str, instance_id: int) -> int:
+        """
+        建立连接。
+
+        API 原型：
+        `int nemu_connect(const wchar_t* path, int index)`
+
+        :param nemu_folder: 模拟器安装路径。
+        :param instance_id: 模拟器实例 ID。
+        :return: 成功返回连接 ID，失败返回 0。
+        """
+        return self.lib.nemu_connect(nemu_folder, instance_id)
+
+    def disconnect(self, connect_id: int) -> None:
+        """
+        断开连接。
+
+        API 原型：
+        `void nemu_disconnect(int handle)`
+
+        :param connect_id: 连接 ID。
+        :return: 无返回值。
+        """
+        return self.lib.nemu_disconnect(connect_id)
+
+    def get_display_id(self, connect_id: int, pkg: str, app_index: int) -> int:
+        """
+        获取指定包的 display id。
+
+        API 原型：
+        `int nemu_get_display_id(int handle, const char* pkg, int appIndex)`
+
+        :param connect_id: 连接 ID。
+        :param pkg: 包名。
+        :param app_index: 多开应用索引。
+        :return: <0 表示失败，>=0 表示有效 display id。
+        """
+        return self.lib.nemu_get_display_id(connect_id, pkg.encode('utf-8'), app_index)
+
+    def capture_display(
+        self,
+        connect_id: int,
+        display_id: int,
+        buf_len: int,
+        width_ptr: ctypes.c_void_p,
+        height_ptr: ctypes.c_void_p,
+        buffer_ptr: ctypes.c_void_p,
+    ) -> int:
+        """
+        截取指定显示屏内容。
+
+        API 原型：
+        `int nemu_capture_display(int handle, unsigned int displayid, int buffer_size, int *width, int *height, unsigned char* pixels)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param buf_len: 缓冲区长度（字节）。
+        :param width_ptr: 用于接收宽度的指针（ctypes.c_void_p/int 指针）。
+        :param height_ptr: 用于接收高度的指针（ctypes.c_void_p/int 指针）。
+        :param buffer_ptr: 用于接收像素数据的指针（ctypes.c_void_p/unsigned char* 指针）。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_capture_display(
+            connect_id,
+            display_id,
+            buf_len,
+            width_ptr,
+            height_ptr,
+            buffer_ptr,
+        )
+
+    def input_text(self, connect_id: int, text: str) -> int:
+        """
+        输入文本。
+
+        API 原型：
+        `int nemu_input_text(int handle, int size, const char* buf)`
+
+        :param connect_id: 连接 ID。
+        :param text: 输入文本（utf-8）。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        buf = text.encode('utf-8')
+        return self.lib.nemu_input_text(connect_id, len(buf), buf)
+
+    def input_touch_down(self, connect_id: int, display_id: int, x: int, y: int) -> int:
+        """
+        发送触摸按下事件。
+
+        API 原型：
+        `int nemu_input_event_touch_down(int handle, int displayid, int x_point, int y_point)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param x: 触摸点 X 坐标。
+        :param y: 触摸点 Y 坐标。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_touch_down(connect_id, display_id, x, y)
+
+    def input_touch_up(self, connect_id: int, display_id: int) -> int:
+        """
+        发送触摸抬起事件。
+
+        API 原型：
+        `int nemu_input_event_touch_up(int handle, int displayid)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_touch_up(connect_id, display_id)
+
+    def input_key_down(self, connect_id: int, display_id: int, key_code: int) -> int:
+        """
+        发送按键按下事件。
+
+        API 原型：
+        `int nemu_input_event_key_down(int handle, int displayid, int key_code)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param key_code: 按键码。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_key_down(connect_id, display_id, key_code)
+
+    def input_key_up(self, connect_id: int, display_id: int, key_code: int) -> int:
+        """
+        发送按键抬起事件。
+
+        API 原型：
+        `int nemu_input_event_key_up(int handle, int displayid, int key_code)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param key_code: 按键码。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_key_up(connect_id, display_id, key_code)
+
+    def input_finger_touch_down(self, connect_id: int, display_id: int, finger_id: int, x: int, y: int) -> int:
+        """
+        多指触摸按下。
+
+        API 原型：
+        `int nemu_input_event_finger_touch_down(int handle, int displayid, int finger_id, int x_point, int y_point)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param finger_id: 手指编号（1-10）。
+        :param x: 触摸点 X 坐标。
+        :param y: 触摸点 Y 坐标。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_finger_touch_down(connect_id, display_id, finger_id, x, y)
+
+    def input_finger_touch_up(self, connect_id: int, display_id: int, finger_id: int) -> int:
+        """
+        多指触摸抬起。
+
+        API 原型：
+        `int nemu_input_event_finger_touch_up(int handle, int displayid, int slot_id)`
+
+        :param connect_id: 连接 ID。
+        :param display_id: 显示屏 ID。
+        :param finger_id: 手指编号（1-10）。
+        :return: 0 表示成功，>0 表示失败。
+        """
+        return self.lib.nemu_input_event_finger_touch_up(connect_id, display_id, finger_id)
+
+    # ------------------------------------------------------------------
+    # 内部工具
+    # ------------------------------------------------------------------
+
+    def __load_dll(self, mumu_root_folder: str) -> ctypes.CDLL:
+        """尝试多条路径加载 DLL。传入为 MuMu 根目录。"""
+        candidate_paths = [
+            os.path.join(mumu_root_folder, "shell", "sdk", "external_renderer_ipc.dll"),
+            os.path.join(
+                mumu_root_folder,
+                "shell",
+                "nx_device",
+                "12.0",
+                "sdk",
+                "external_renderer_ipc.dll",
+            ),
+        ]
+        for p in candidate_paths:
+            if not os.path.exists(p):
+                continue
+            try:
+                return ctypes.CDLL(p)
+            except OSError as e:  # pragma: no cover
+                logger.warning("Failed to load DLL (%s): %s", p, e)
+        raise NemuIpcIncompatible("external_renderer_ipc.dll not found or failed to load.")
+
+    def __declare_prototypes(self) -> None:
+        """声明 DLL 函数原型，确保 ctypes 类型安全。"""
+        # 连接 / 断开
+        self.lib.nemu_connect.argtypes = [ctypes.c_wchar_p, ctypes.c_int]
+        self.lib.nemu_connect.restype = ctypes.c_int
+
+        self.lib.nemu_disconnect.argtypes = [ctypes.c_int]
+        self.lib.nemu_disconnect.restype = None
+
+        # 获取 display id
+        self.lib.nemu_get_display_id.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
+        self.lib.nemu_get_display_id.restype = ctypes.c_int
+
+        # 截图
+        self.lib.nemu_capture_display.argtypes = [
+            ctypes.c_int,
+            ctypes.c_uint,
+            ctypes.c_int,
+            ctypes.c_void_p,
+            ctypes.c_void_p,
+            ctypes.c_void_p,
+        ]
+        self.lib.nemu_capture_display.restype = ctypes.c_int
+
+        # 输入文本
+        self.lib.nemu_input_text.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_char_p]
+        self.lib.nemu_input_text.restype = ctypes.c_int
+
+        # 触摸
+        self.lib.nemu_input_event_touch_down.argtypes = [
+            ctypes.c_int,
+            ctypes.c_int,
+            ctypes.c_int,
+            ctypes.c_int,
+        ]
+        self.lib.nemu_input_event_touch_down.restype = ctypes.c_int
+
+        self.lib.nemu_input_event_touch_up.argtypes = [ctypes.c_int, ctypes.c_int]
+        self.lib.nemu_input_event_touch_up.restype = ctypes.c_int
+
+        # 按键
+        self.lib.nemu_input_event_key_down.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
+        self.lib.nemu_input_event_key_down.restype = ctypes.c_int
+
+        self.lib.nemu_input_event_key_up.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
+        self.lib.nemu_input_event_key_up.restype = ctypes.c_int
+
+        # 多指触摸
+        self.lib.nemu_input_event_finger_touch_down.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int]
+        self.lib.nemu_input_event_finger_touch_down.restype = ctypes.c_int
+
+        self.lib.nemu_input_event_finger_touch_up.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
+        self.lib.nemu_input_event_finger_touch_up.restype = ctypes.c_int
+
+        logger.debug("DLL function prototypes declared")