uiautodev 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

uiautodev/model.py

@@ -33,13 +33,17 @@ class Rect(BaseModel):
 
 class Node(BaseModel):
     key: str
-    name: str
+    name: str # can be seen as description
     bounds: Optional[Tuple[float, float, float, float]] = None
     rect: Optional[Rect] = None
-    properties: Dict[str, Union[str, bool]] = []
+    properties: Dict[str, Union[str, bool]] = {}
     children: List[Node] = []
 
 
+class OCRNode(Node):
+    confidence: float
+
+
 class WindowSize(typing.NamedTuple):
     width: int
     height: int

uiautodev/provider.py

@@ -12,6 +12,7 @@ import adbutils
 
 from uiautodev.driver.android import AndroidDriver
 from uiautodev.driver.base_driver import BaseDriver
+from uiautodev.driver.harmony import HDC, HarmonyDriver
 from uiautodev.driver.ios import IOSDriver
 from uiautodev.driver.mock import MockDriver
 from uiautodev.exceptions import UiautoException
@@ -27,7 +28,7 @@ class BaseProvider(abc.ABC):
     @abc.abstractmethod
     def get_device_driver(self, serial: str) -> BaseDriver:
         raise NotImplementedError()
-    
+
     def get_single_device_driver(self) -> BaseDriver:
         """ debug use """
         devs = self.list_devices()
@@ -66,11 +67,25 @@ class IOSProvider(BaseProvider):
     @lru_cache
     def get_device_driver(self, serial: str) -> BaseDriver:
         return IOSDriver(serial)
-    
+
+
+class HarmonyProvider(BaseProvider):
+    def __init__(self):
+        super().__init__()
+        self.hdc = HDC()
+
+    def list_devices(self) -> list[DeviceInfo]:
+        devices = self.hdc.list_device()
+        return [DeviceInfo(serial=d, model=self.hdc.get_model(d), name=self.hdc.get_model(d)) for d in devices]
+
+    @lru_cache
+    def get_device_driver(self, serial: str) -> HarmonyDriver:
+        return HarmonyDriver(self.hdc, serial)
+
 
 class MockProvider(BaseProvider):
     def list_devices(self) -> list[DeviceInfo]:
         return [DeviceInfo(serial="mock-serial", model="mock-model", name="mock-name")]
 
     def get_device_driver(self, serial: str) -> BaseDriver:
-        return MockDriver(serial)
+        return MockDriver(serial)

uiautodev/remote/android_input.py

@@ -0,0 +1,74 @@
+# Ref
+# https://github.com/Genymobile/scrcpy/blob/master/app/src/android/input.h
+from enum import IntEnum
+
+
+class MetaState(IntEnum):
+    """Android meta state flags ported from Android's KeyEvent class
+    
+    These flags represent the state of meta keys such as ALT, SHIFT, CTRL, etc.
+    They can be combined using bitwise OR operations to represent multiple
+    meta keys being pressed simultaneously.
+    
+    The values and comments are taken directly from the Android source code
+    to maintain compatibility and provide accurate descriptions.
+    """
+    # No meta keys are pressed
+    NONE = 0x0
+    
+    # This mask is used to check whether one of the SHIFT meta keys is pressed
+    SHIFT_ON = 0x1
+    
+    # This mask is used to check whether one of the ALT meta keys is pressed
+    ALT_ON = 0x2
+    
+    # This mask is used to check whether the SYM meta key is pressed
+    SYM_ON = 0x4
+    
+    # This mask is used to check whether the FUNCTION meta key is pressed
+    FUNCTION_ON = 0x8
+    
+    # This mask is used to check whether the left ALT meta key is pressed
+    ALT_LEFT_ON = 0x10
+    
+    # This mask is used to check whether the right ALT meta key is pressed
+    ALT_RIGHT_ON = 0x20
+    
+    # This mask is used to check whether the left SHIFT meta key is pressed
+    SHIFT_LEFT_ON = 0x40
+    
+    # This mask is used to check whether the right SHIFT meta key is pressed
+    SHIFT_RIGHT_ON = 0x80
+    
+    # This mask is used to check whether the CAPS LOCK meta key is on
+    CAPS_LOCK_ON = 0x100000
+    
+    # This mask is used to check whether the NUM LOCK meta key is on
+    NUM_LOCK_ON = 0x200000
+    
+    # This mask is used to check whether the SCROLL LOCK meta key is on
+    SCROLL_LOCK_ON = 0x400000
+    
+    # This mask is used to check whether one of the CTRL meta keys is pressed
+    CTRL_ON = 0x1000
+    
+    # This mask is used to check whether the left CTRL meta key is pressed
+    CTRL_LEFT_ON = 0x2000
+    
+    # This mask is used to check whether the right CTRL meta key is pressed
+    CTRL_RIGHT_ON = 0x4000
+    
+    # This mask is used to check whether one of the META meta keys is pressed
+    META_ON = 0x10000
+    
+    # This mask is used to check whether the left META meta key is pressed
+    META_LEFT_ON = 0x20000
+    
+    # This mask is used to check whether the right META meta key is pressed
+    META_RIGHT_ON = 0x40000
+
+
+class KeyeventAction(IntEnum):
+    DOWN = 0
+    UP = 1
+    MULTIPLE = 2

uiautodev/remote/keycode.py

@@ -0,0 +1,350 @@
+from enum import IntEnum
+
+
+class KeyCode(IntEnum):
+    """Android key codes ported from Android's KeyEvent class
+    
+    This enum contains all the key codes defined in Android's KeyEvent class,
+    which are used for sending key events to Android devices through scrcpy.
+    
+    The comments for each key code are taken directly from the Android source code
+    to maintain compatibility and provide accurate descriptions.
+    """
+    # Unknown key code
+    UNKNOWN = 0
+    # Soft Left key - Usually situated below the display on phones
+    SOFT_LEFT = 1
+    # Soft Right key - Usually situated below the display on phones
+    SOFT_RIGHT = 2
+    # Home key - This key is handled by the framework and is never delivered to applications
+    HOME = 3
+    # Back key
+    BACK = 4
+    # Call key
+    CALL = 5
+    # End Call key
+    ENDCALL = 6
+    # '0' key
+    KEY_0 = 7
+    # '1' key
+    KEY_1 = 8
+    # '2' key
+    KEY_2 = 9
+    # '3' key
+    KEY_3 = 10
+    # '4' key
+    KEY_4 = 11
+    # '5' key
+    KEY_5 = 12
+    # '6' key
+    KEY_6 = 13
+    # '7' key
+    KEY_7 = 14
+    # '8' key
+    KEY_8 = 15
+    # '9' key
+    KEY_9 = 16
+    # '*' key
+    STAR = 17
+    # '#' key
+    POUND = 18
+    # Directional Pad Up key - May also be synthesized from trackball motions
+    DPAD_UP = 19
+    # Directional Pad Down key - May also be synthesized from trackball motions
+    DPAD_DOWN = 20
+    # Directional Pad Left key - May also be synthesized from trackball motions
+    DPAD_LEFT = 21
+    # Directional Pad Right key - May also be synthesized from trackball motions
+    DPAD_RIGHT = 22
+    # Directional Pad Center key - May also be synthesized from trackball motions
+    DPAD_CENTER = 23
+    # Volume Up key - Adjusts the speaker volume up
+    VOLUME_UP = 24
+    # Volume Down key - Adjusts the speaker volume down
+    VOLUME_DOWN = 25
+    # Power key
+    POWER = 26
+    # Camera key - Used to launch a camera application or take pictures
+    CAMERA = 27
+    # Clear key
+    CLEAR = 28
+    A = 29
+    B = 30
+    C = 31
+    D = 32
+    E = 33
+    F = 34
+    G = 35
+    H = 36
+    I = 37
+    J = 38
+    K = 39
+    L = 40
+    M = 41
+    N = 42
+    O = 43
+    P = 44
+    Q = 45
+    R = 46
+    S = 47
+    T = 48
+    U = 49
+    V = 50
+    W = 51
+    X = 52
+    Y = 53
+    Z = 54
+    COMMA = 55
+    PERIOD = 56
+    ALT_LEFT = 57
+    ALT_RIGHT = 58
+    SHIFT_LEFT = 59
+    SHIFT_RIGHT = 60
+    TAB = 61
+    SPACE = 62
+    SYM = 63
+    EXPLORER = 64
+    ENVELOPE = 65
+    # Enter key
+    ENTER = 66
+    # Backspace key - Deletes characters before the insertion point
+    DEL = 67
+    GRAVE = 68
+    MINUS = 69
+    EQUALS = 70
+    LEFT_BRACKET = 71
+    RIGHT_BRACKET = 72
+    BACKSLASH = 73
+    SEMICOLON = 74
+    APOSTROPHE = 75
+    SLASH = 76
+    AT = 77
+    NUM = 78
+    HEADSETHOOK = 79
+    FOCUS = 80
+    PLUS = 81
+    # Menu key
+    MENU = 82
+    NOTIFICATION = 83
+    SEARCH = 84
+    MEDIA_PLAY_PAUSE = 85
+    MEDIA_STOP = 86
+    MEDIA_NEXT = 87
+    MEDIA_PREVIOUS = 88
+    MEDIA_REWIND = 89
+    MEDIA_FAST_FORWARD = 90
+    MUTE = 91
+    PAGE_UP = 92
+    PAGE_DOWN = 93
+    PICTSYMBOLS = 94
+    SWITCH_CHARSET = 95
+    BUTTON_A = 96
+    BUTTON_B = 97
+    BUTTON_C = 98
+    BUTTON_X = 99
+    BUTTON_Y = 100
+    BUTTON_Z = 101
+    BUTTON_L1 = 102
+    BUTTON_R1 = 103
+    BUTTON_L2 = 104
+    BUTTON_R2 = 105
+    BUTTON_THUMBL = 106
+    BUTTON_THUMBR = 107
+    BUTTON_START = 108
+    BUTTON_SELECT = 109
+    BUTTON_MODE = 110
+    ESCAPE = 111
+    FORWARD_DEL = 112
+    CTRL_LEFT = 113
+    CTRL_RIGHT = 114
+    CAPS_LOCK = 115
+    SCROLL_LOCK = 116
+    META_LEFT = 117
+    META_RIGHT = 118
+    FUNCTION = 119
+    SYSRQ = 120
+    BREAK = 121
+    MOVE_HOME = 122
+    MOVE_END = 123
+    INSERT = 124
+    FORWARD = 125
+    MEDIA_PLAY = 126
+    MEDIA_PAUSE = 127
+    MEDIA_CLOSE = 128
+    MEDIA_EJECT = 129
+    MEDIA_RECORD = 130
+    F1 = 131
+    F2 = 132
+    F3 = 133
+    F4 = 134
+    F5 = 135
+    F6 = 136
+    F7 = 137
+    F8 = 138
+    F9 = 139
+    F10 = 140
+    F11 = 141
+    F12 = 142
+    NUM_LOCK = 143
+    NUMPAD_0 = 144
+    NUMPAD_1 = 145
+    NUMPAD_2 = 146
+    NUMPAD_3 = 147
+    NUMPAD_4 = 148
+    NUMPAD_5 = 149
+    NUMPAD_6 = 150
+    NUMPAD_7 = 151
+    NUMPAD_8 = 152
+    NUMPAD_9 = 153
+    NUMPAD_DIVIDE = 154
+    NUMPAD_MULTIPLY = 155
+    NUMPAD_SUBTRACT = 156
+    NUMPAD_ADD = 157
+    NUMPAD_DOT = 158
+    NUMPAD_COMMA = 159
+    NUMPAD_ENTER = 160
+    NUMPAD_EQUALS = 161
+    NUMPAD_LEFT_PAREN = 162
+    NUMPAD_RIGHT_PAREN = 163
+    VOLUME_MUTE = 164
+    INFO = 165
+    CHANNEL_UP = 166
+    CHANNEL_DOWN = 167
+    ZOOM_IN = 168
+    ZOOM_OUT = 169
+    TV = 170
+    WINDOW = 171
+    GUIDE = 172
+    DVR = 173
+    BOOKMARK = 174
+    CAPTIONS = 175
+    SETTINGS = 176
+    TV_POWER = 177
+    TV_INPUT = 178
+    STB_POWER = 179
+    STB_INPUT = 180
+    AVR_POWER = 181
+    AVR_INPUT = 182
+    PROG_RED = 183
+    PROG_GREEN = 184
+    PROG_YELLOW = 185
+    PROG_BLUE = 186
+    APP_SWITCH = 187
+    BUTTON_1 = 188
+    BUTTON_2 = 189
+    BUTTON_3 = 190
+    BUTTON_4 = 191
+    BUTTON_5 = 192
+    BUTTON_6 = 193
+    BUTTON_7 = 194
+    BUTTON_8 = 195
+    BUTTON_9 = 196
+    BUTTON_10 = 197
+    BUTTON_11 = 198
+    BUTTON_12 = 199
+    BUTTON_13 = 200
+    BUTTON_14 = 201
+    BUTTON_15 = 202
+    BUTTON_16 = 203
+    LANGUAGE_SWITCH = 204
+    MANNER_MODE = 205
+    MODE_3D = 206
+    CONTACTS = 207
+    CALENDAR = 208
+    MUSIC = 209
+    CALCULATOR = 210
+    ZENKAKU_HANKAKU = 211
+    EISU = 212
+    MUHENKAN = 213
+    HENKAN = 214
+    KATAKANA_HIRAGANA = 215
+    YEN = 216
+    RO = 217
+    KANA = 218
+    ASSIST = 219
+    BRIGHTNESS_DOWN = 220
+    BRIGHTNESS_UP = 221
+    MEDIA_AUDIO_TRACK = 222
+    SLEEP = 223
+    WAKEUP = 224
+    PAIRING = 225
+    MEDIA_TOP_MENU = 226
+    KEY_11 = 227
+    KEY_12 = 228
+    LAST_CHANNEL = 229
+    TV_DATA_SERVICE = 230
+    VOICE_ASSIST = 231
+    TV_RADIO_SERVICE = 232
+    TV_TELETEXT = 233
+    TV_NUMBER_ENTRY = 234
+    TV_TERRESTRIAL_ANALOG = 235
+    TV_TERRESTRIAL_DIGITAL = 236
+    TV_SATELLITE = 237
+    TV_SATELLITE_BS = 238
+    TV_SATELLITE_CS = 239
+    TV_SATELLITE_SERVICE = 240
+    TV_NETWORK = 241
+    TV_ANTENNA_CABLE = 242
+    TV_INPUT_HDMI_1 = 243
+    TV_INPUT_HDMI_2 = 244
+    TV_INPUT_HDMI_3 = 245
+    TV_INPUT_HDMI_4 = 246
+    TV_INPUT_COMPOSITE_1 = 247
+    TV_INPUT_COMPOSITE_2 = 248
+    TV_INPUT_COMPONENT_1 = 249
+    TV_INPUT_COMPONENT_2 = 250
+    TV_INPUT_VGA_1 = 251
+    TV_AUDIO_DESCRIPTION = 252
+    TV_AUDIO_DESCRIPTION_MIX_UP = 253
+    TV_AUDIO_DESCRIPTION_MIX_DOWN = 254
+    TV_ZOOM_MODE = 255
+    TV_CONTENTS_MENU = 256
+    TV_MEDIA_CONTEXT_MENU = 257
+    TV_TIMER_PROGRAMMING = 258
+    HELP = 259
+    NAVIGATE_PREVIOUS = 260
+    NAVIGATE_NEXT = 261
+    NAVIGATE_IN = 262
+    NAVIGATE_OUT = 263
+    STEM_PRIMARY = 264
+    STEM_1 = 265
+    STEM_2 = 266
+    STEM_3 = 267
+    DPAD_UP_LEFT = 268
+    DPAD_DOWN_LEFT = 269
+    DPAD_UP_RIGHT = 270
+    DPAD_DOWN_RIGHT = 271
+    MEDIA_SKIP_FORWARD = 272
+    MEDIA_SKIP_BACKWARD = 273
+    MEDIA_STEP_FORWARD = 274
+    MEDIA_STEP_BACKWARD = 275
+    SOFT_SLEEP = 276
+    CUT = 277
+    COPY = 278
+    PASTE = 279
+    SYSTEM_NAVIGATION_UP = 280
+    SYSTEM_NAVIGATION_DOWN = 281
+    SYSTEM_NAVIGATION_LEFT = 282
+    SYSTEM_NAVIGATION_RIGHT = 283
+    ALL_APPS = 284
+    
+    # =========================================================================
+    # Aliases for original Android KeyEvent names
+    # =========================================================================
+    # These aliases are provided to maintain compatibility with the original
+    # Android KeyEvent naming convention (AKEYCODE_*). This makes it easier
+    # to reference keys using the same names as in Android documentation.
+    
+    # Numeric key aliases
+    KEYCODE_0 = KEY_0         # '0' key
+    KEYCODE_1 = KEY_1         # '1' key
+    KEYCODE_2 = KEY_2         # '2' key
+    KEYCODE_3 = KEY_3         # '3' key
+    KEYCODE_4 = KEY_4         # '4' key
+    KEYCODE_5 = KEY_5         # '5' key
+    KEYCODE_6 = KEY_6         # '6' key
+    KEYCODE_7 = KEY_7         # '7' key
+    KEYCODE_8 = KEY_8         # '8' key
+    KEYCODE_9 = KEY_9         # '9' key
+    KEYCODE_11 = KEY_11       # '11' key
+    KEYCODE_12 = KEY_12       # '12' key

uiautodev/remote/scrcpy.py

@@ -0,0 +1,177 @@
+import asyncio
+import json
+import logging
+import os
+import socket
+import struct
+from typing import Optional
+
+import retry
+from adbutils import AdbError, Network, adb
+from adbutils._adb import AdbConnection
+from adbutils._device import AdbDevice
+from starlette.websockets import WebSocket, WebSocketDisconnect
+
+from uiautodev.remote.touch_controller import ScrcpyTouchController
+
+logger = logging.getLogger(__name__)
+
+class ScrcpyServer:
+    """
+    ScrcpyServer class is responsible for managing the scrcpy server on Android devices.
+    It handles the initialization, communication, and control of the scrcpy server,
+    including video streaming and touch control.
+    """
+
+    def __init__(self, device: AdbDevice, scrcpy_jar_path: Optional[str] = None):
+        """
+        Initializes the ScrcpyServer instance.
+
+        Args:
+            scrcpy_jar_path (str, optional): Path to the scrcpy server JAR file. Defaults to None.
+        """
+        self.scrcpy_jar_path = scrcpy_jar_path or os.path.join(os.path.dirname(__file__), '../binaries/scrcpy_server.jar')
+        self.device = device
+        self.resolution_width = 0  # scrcpy 投屏转换宽度
+        self.resolution_height = 0  # scrcpy 投屏转换高度
+        
+        self._shell_conn: AdbConnection
+        self._video_conn: socket.socket
+        self._control_conn: socket.socket
+
+        self._setup_connection()
+    
+    def _setup_connection(self):
+        self._shell_conn = self._start_scrcpy_server(control=True)
+        self._video_conn = self._connect_scrcpy(self.device)
+        self._control_conn = self._connect_scrcpy(self.device)
+        self._parse_scrcpy_info(self._video_conn)
+        self.controller = ScrcpyTouchController(self._control_conn)
+
+    @retry.retry(exceptions=AdbError, tries=20, delay=0.1)
+    def _connect_scrcpy(self, device: AdbDevice) -> socket.socket:
+        return device.create_connection(Network.LOCAL_ABSTRACT, 'scrcpy')
+
+    def _parse_scrcpy_info(self, conn: socket.socket):
+        dummy_byte = conn.recv(1)
+        if not dummy_byte or dummy_byte != b"\x00":
+            raise ConnectionError("Did not receive Dummy Byte!")
+        logger.debug('Received Dummy Byte!')
+        device_name = conn.recv(64).decode('utf-8').rstrip('\x00')
+        logger.debug(f'Device name: {device_name}')
+        codec = conn.recv(4)
+        logger.debug(f'resolution_data: {codec}')
+        resolution_data = conn.recv(8)
+        logger.debug(f'resolution_data: {resolution_data}')
+        self.resolution_width, self.resolution_height = struct.unpack(">II", resolution_data)
+        logger.debug(f'Resolution: {self.resolution_width}x{self.resolution_height}')
+
+    def close(self):
+        try:
+            self._control_conn.close()
+            self._video_conn.close()
+            self._shell_conn.close()
+        except:
+            pass
+    
+    def __del__(self):
+        self.close()
+
+    def _start_scrcpy_server(self, control: bool = True) -> AdbConnection:
+        """
+        Pushes the scrcpy server JAR file to the Android device and starts the scrcpy server.
+
+        Args:
+            control (bool, optional): Whether to enable touch control. Defaults to True.
+
+        Returns:
+            AdbConnection
+        """
+        # 获取设备对象
+        device = self.device
+
+        # 推送 scrcpy 服务器到设备
+        device.sync.push(self.scrcpy_jar_path, '/data/local/tmp/scrcpy_server.jar', check=True)
+        logger.info('scrcpy server JAR pushed to device')
+
+        # 构建启动 scrcpy 服务器的命令
+        start_command = (
+            'CLASSPATH=/data/local/tmp/scrcpy_server.jar '
+            'app_process / '
+            'com.genymobile.scrcpy.Server 2.7 '
+            'log_level=info max_size=1024 max_fps=30 '
+            'video_bit_rate=8000000 tunnel_forward=true '
+            'send_frame_meta=false '
+            f'control={"true" if control else "false"} '
+            'audio=false show_touches=false stay_awake=false '
+            'power_off_on_close=false clipboard_autosync=false'
+        )
+        conn = device.shell(start_command, stream=True)
+        logger.debug("scrcpy output: %s", conn.conn.recv(100))
+        return conn # type: ignore
+
+    async def handle_unified_websocket(self, websocket: WebSocket, serial=''):
+        logger.info(f"[Unified] WebSocket connection from {websocket} for serial: {serial}")
+
+        video_task = asyncio.create_task(self._stream_video_to_websocket(self._video_conn, websocket))
+        control_task  = asyncio.create_task(self._handle_control_websocket(websocket))
+        
+        try:
+            # 不使用 return_exceptions=True，让异常能够正确传播
+            await asyncio.gather(video_task, control_task)
+        finally:
+            # 取消任务
+            for task in (video_task, control_task):
+                if not task.done():
+                    task.cancel()
+            logger.info(f"[Unified] WebSocket closed for serial={serial}")
+
+    async def _stream_video_to_websocket(self, conn: socket.socket, ws: WebSocket):
+        # Set socket to non-blocking mode
+        conn.setblocking(False)
+        
+        while True:
+            # check if ws closed
+            if ws.client_state.name != "CONNECTED":
+                logger.info('WebSocket no longer connected. Exiting video stream.')
+                break
+            # Use asyncio to read data asynchronously
+            data = await asyncio.get_event_loop().sock_recv(conn, 1024*1024)
+            if not data:
+                logger.warning('No data received, connection may be closed.')
+                raise ConnectionError("Video stream ended unexpectedly")
+            # send data to ws
+            await ws.send_bytes(data)
+
+    async def _handle_control_websocket(self, ws: WebSocket):
+        while True:
+            try:
+                message = await ws.receive_text()
+                logger.debug(f"[Unified] Received message: {message}")
+                message = json.loads(message)
+
+                width, height = self.resolution_width, self.resolution_height
+                message_type = message.get('type')
+                if message_type == 'touchMove':
+                    xP = message['xP']
+                    yP = message['yP']
+                    self.controller.move(int(xP * width), int(yP * height), width, height)
+                elif message_type == 'touchDown':
+                    xP = message['xP']
+                    yP = message['yP']
+                    self.controller.down(int(xP * width), int(yP * height), width, height)
+                elif message_type == 'touchUp':
+                    xP = message['xP']
+                    yP = message['yP']
+                    self.controller.up(int(xP * width), int(yP * height), width, height)
+                elif message_type == 'keyEvent':
+                    event_number = message['data']['eventNumber']
+                    self.device.shell(f'input keyevent {event_number}')
+                elif message_type == 'text':
+                    text = message['detail']
+                    self.device.shell(f'am broadcast -a SONIC_KEYBOARD --es msg \'{text}\'')
+                elif message_type == 'ping':
+                    await ws.send_text(json.dumps({"type": "pong"}))
+            except json.JSONDecodeError as e:
+                logger.error(f"Invalid JSON message: {e}")
+                continue