robotframework-desktop 0.1.0__py3-none-any.whl

DesktopLibrary/keywords/keyboard.py

@@ -0,0 +1,171 @@
+import time
+
+import pyautogui
+import pyperclip
+from robot.api.deco import keyword
+
+from .base import KeywordGroup, busy_action
+
+_KEY_ALIASES = {
+    'ESCAPE': 'esc',
+    'ESC': 'esc',
+    'RETURN': 'enter',
+    'ENTER': 'enter',
+    'BACKSPACE': 'backspace',
+    'DELETE': 'delete',
+    'DEL': 'delete',
+    'PAGE_UP': 'pageup',
+    'PAGEUP': 'pageup',
+    'PAGE_DOWN': 'pagedown',
+    'PAGEDOWN': 'pagedown',
+    'CTRL': 'ctrl',
+    'CONTROL': 'ctrl',
+    'ALT': 'alt',
+    'SHIFT': 'shift',
+    'WIN': 'win',
+    'META': 'win',
+    'CMD': 'win',
+    'WINDOWS': 'win',
+    'CAPS_LOCK': 'capslock',
+    'NUM_LOCK': 'numlock',
+    'SCROLL_LOCK': 'scrolllock',
+    'PRINTSCREEN': 'printscreen',
+    'PRINT': 'printscreen',
+    'INSERT': 'insert',
+    'SPACE': 'space',
+    'TAB': 'tab',
+    'HOME': 'home',
+    'END': 'end',
+    'UP': 'up',
+    'DOWN': 'down',
+    'LEFT': 'left',
+    'RIGHT': 'right',
+    'PAUSE': 'pause',
+    'ALTGR': 'altright',
+}
+
+
+def to_key(name: str) -> str:
+    text = str(name).strip()
+    if len(text) == 1:
+        return text.lower()
+    upper = text.upper().replace(' ', '_')
+    if upper in _KEY_ALIASES:
+        return _KEY_ALIASES[upper]
+    key = upper.replace('_', '').lower()
+    if key in pyautogui.KEYBOARD_KEYS:
+        return key
+    raise ValueError(f"Unknown key '{name}'.")
+
+
+class KeyboardKeywords(KeywordGroup):
+    def _focus(self, element):
+        if element not in (None, ''):
+            self._click_at(self._locate(element))
+            time.sleep(0.05)
+
+    def _write(self, text: str):
+        if text.isascii():
+            pyautogui.write(text, interval=0.01)
+        else:
+            self._paste(text)
+
+    @staticmethod
+    def _paste(text: str):
+        try:
+            previous = pyperclip.paste()
+        except Exception:
+            previous = None
+        pyperclip.copy(text)
+        pyautogui.hotkey('ctrl', 'v')
+        time.sleep(0.1)
+        if previous is not None:
+            pyperclip.copy(previous)
+
+    @keyword
+    @busy_action
+    def input_text(self, element, text: str):
+        """Clicks ``element`` (if given) and types ``text``.
+
+        ``element`` can be empty to type into the currently focused control.
+        Non-ASCII text is typed through the clipboard.
+
+        Examples:
+        | Input Text | username_field.png | john |
+        | Input Text | automation_id:txtName | John Doe |
+        | Input Text | ${EMPTY} | text typed into the focused field |
+        """
+        self._focus(element)
+        self._ready()
+        self._write(str(text))
+
+    @keyword
+    @busy_action
+    def paste_text(self, element, text: str):
+        """Clicks ``element`` (if given) and pastes ``text`` via the clipboard.
+
+        Examples:
+        | Paste Text | notes.png | Olá, ação! |
+        | Paste Text | ${EMPTY} | pasted into the focused control |
+        """
+        self._focus(element)
+        self._ready()
+        self._paste(str(text))
+
+    @keyword
+    @busy_action
+    def press_special_key(self, key_constant: str):
+        """Presses a special key such as ``ENTER``, ``TAB``, ``ESC``, ``DELETE``,
+        ``F1``...``F12``, ``UP``, ``DOWN``, ``PAGE_UP``, ``HOME``.
+
+        Examples:
+        | Double Click | field.png |
+        | Press Special Key | DELETE |
+        | Press Special Key | ENTER |
+        """
+        self._ready()
+        pyautogui.press(to_key(key_constant))
+
+    @keyword
+    @busy_action
+    def key_down(self, key_constant: str):
+        """Presses and holds a keyboard key.
+
+        Example:
+        | Key Down | CTRL |
+        | Click | item1.png |
+        | Click | item2.png |
+        | Key Up | CTRL |
+        """
+        self._ready()
+        pyautogui.keyDown(to_key(key_constant))
+
+    @keyword
+    @busy_action
+    def key_up(self, key_constant: str):
+        """Releases a keyboard key held by `Key Down`.
+
+        Example:
+        | Key Up | CTRL |
+        """
+        pyautogui.keyUp(to_key(key_constant))
+
+    @keyword
+    @busy_action
+    def type_with_modifiers(self, text: str, *modifiers):
+        """Types ``text`` while holding ``modifiers`` (``CTRL``, ``ALT``, ``SHIFT``, ``WIN``).
+
+        Examples:
+        | Type With Modifiers | a | CTRL | | # select all |
+        | Type With Modifiers | s | CTRL | SHIFT |
+        """
+        mods = [to_key(m) for m in modifiers]
+        self._ready()
+        for m in mods:
+            pyautogui.keyDown(m)
+        try:
+            for ch in str(text):
+                pyautogui.press(to_key(ch) if len(ch) == 1 else ch)
+        finally:
+            for m in reversed(mods):
+                pyautogui.keyUp(m)

DesktopLibrary/keywords/mouse.py

@@ -0,0 +1,357 @@
+import time
+
+import pyautogui
+from robot.api.deco import keyword
+
+from ..geometry import Region, sort_matches, to_region
+from ..locator import parse_locator
+from .base import KeywordGroup, busy_action
+
+_BUTTONS = {'LEFT': 'left', 'MIDDLE': 'middle', 'RIGHT': 'right'}
+
+
+class MouseKeywords(KeywordGroup):
+    def __init__(self, ctx):
+        super().__init__(ctx)
+        self._held: set[str] = set()
+
+    # ---- clicks -------------------------------------------------------------
+    @keyword
+    @busy_action
+    def click(self, element, x_offset: int = 0, y_offset: int = 0):
+        """Clicks the given ``element``.
+
+        ``element`` can be any of the formats described in `Element locators`
+        (image path, dictionary, UI Automation locator, region or match).
+        ``x_offset``/``y_offset`` are added to the offset defined in the locator.
+
+        Examples:
+        | Click | buttons/save.png |
+        | Click | save.png=0.9 | | | # custom similarity |
+        | Click | save.png | 10 | -5 | # click 10px right and 5px above the center |
+        | Click | automation_id:btnSave |
+        | Click | name:OK AND control_type:Button |
+        | ${btn}= | Create Dictionary | path=ok.png | similarity=0.8 |
+        | Click | ${btn} |
+        | Click | ${{ {"path": ["ok.png", "ok_dark.png"], "offset": {"x": 5, "y": 0}} }} |
+        """
+        self._click_at(self._locate(element, x_offset, y_offset))
+
+    @keyword
+    @busy_action
+    def double_click(self, element, x_offset: int = 0, y_offset: int = 0):
+        """Double clicks the given ``element``. See `Click` for arguments.
+
+        Examples:
+        | Double Click | file_icon.png |
+        | Double Click | name:report.txt |
+        """
+        self._click_at(self._locate(element, x_offset, y_offset), clicks=2)
+
+    @keyword
+    @busy_action
+    def right_click(self, element, x_offset: int = 0, y_offset: int = 0):
+        """Right clicks the given ``element``. See `Click` for arguments.
+
+        Examples:
+        | Right Click | desktop_icon.png |
+        | Right Click | automation_id:listItems | 0 | 20 |
+        """
+        self._click_at(
+            self._locate(element, x_offset, y_offset), button='right'
+        )
+
+    def _click_in(self, area, target, clicks=1, button='left'):
+        area_match = self._locate(area)
+        locator = parse_locator(target).with_region(
+            Region(*area_match.to_tuple())
+        )
+        self._click_at(self._locate(locator), clicks, button)
+
+    @keyword
+    @busy_action
+    def click_in(self, area_element, target_element):
+        """Clicks ``target_element`` searched only inside ``area_element``.
+
+        Examples:
+        | Click In | dialog.png | ok.png |
+        | Click In | name:Settings | checkbox.png |
+        """
+        self._click_in(area_element, target_element)
+
+    @keyword
+    @busy_action
+    def double_click_in(self, area_element, target_element):
+        """Double clicks ``target_element`` inside ``area_element``.
+
+        Example:
+        | Double Click In | file_list.png | report_icon.png |
+        """
+        self._click_in(area_element, target_element, clicks=2)
+
+    @keyword
+    @busy_action
+    def right_click_in(self, area_element, target_element):
+        """Right clicks ``target_element`` inside ``area_element``.
+
+        Example:
+        | Right Click In | file_list.png | report_icon.png |
+        """
+        self._click_in(area_element, target_element, button='right')
+
+    @keyword
+    @busy_action
+    def click_nth(self, element, index: int, sort_by_column: bool = True):
+        """Clicks the ``index``-th (0 based) occurrence of ``element``.
+
+        Matches are sorted by column (left to right, then top to bottom) when
+        ``sort_by_column`` is true, otherwise by row (top to bottom, then left
+        to right).
+
+        Examples:
+        | Click Nth | checkbox.png | 0 |
+        | Click Nth | checkbox.png | 2 | ${FALSE} |
+        """
+        matches = self.ctx.find_all(
+            element, self.ctx.timeout, wait_cursor=True
+        )
+        if not matches:
+            raise AssertionError(
+                f"Element '{parse_locator(element).describe()}' not found."
+            )
+        matches = sort_matches(matches, 'column' if sort_by_column else 'row')
+        index = int(index)
+        if not 0 <= index < len(matches):
+            raise AssertionError(
+                f'Index {index} out of range: {len(matches)} matches found.'
+            )
+        self.ctx.show_action(matches[index])
+        self._click_at(matches[index])
+
+    @keyword
+    @busy_action
+    def click_on_match(self, match):
+        """Clicks a match returned by `Return Match From Region` (no offset).
+
+        Example:
+        | ${match}= | Return Match From Region | ${region} | item.png |
+        | Click On Match | ${match} |
+        """
+        self._click_at(self._locate(match))
+
+    @keyword
+    @busy_action
+    def double_click_on_match(self, match):
+        """Double clicks a match returned by `Return Match From Region`.
+
+        Example:
+        | Double Click On Match | ${match} |
+        """
+        self._click_at(self._locate(match), clicks=2)
+
+    @keyword
+    @busy_action
+    def click_on_region(self, region):
+        """Clicks the center of ``region`` (e.g. from `Get Extended Region From`).
+
+        Example:
+        | ${region}= | Get Extended Region From | label.png | below | 1 |
+        | Click On Region | ${region} |
+        """
+        self._click_at(self._locate(to_region(region)))
+
+    @keyword
+    @busy_action
+    def double_click_on_region(self, region):
+        """Double clicks the center of ``region``.
+
+        Example:
+        | Double Click On Region | ${region} |
+        """
+        self._click_at(self._locate(to_region(region)), clicks=2)
+
+    @keyword
+    @busy_action
+    def click_region(
+        self, coordinates, wait_change: float = 0, timeout: float = 0
+    ):
+        """Clicks the center of the region ``coordinates`` ``[x, y, w, h]``.
+
+        If ``wait_change`` is greater than 0, waits up to that many seconds for
+        the region content to change after the click and fails otherwise.
+        If ``timeout`` is greater than 0 the region is highlighted for that
+        many seconds before clicking.
+
+        Examples:
+        | ${coor}= | Create List | 0 | 0 | 100 | 100 |
+        | Click Region | ${coor} |
+        | Click Region | ${coor} | 2 | | # wait up to 2s for the region to change |
+        | Click Region | ${coor} | 0 | 1 | # highlight 1s before clicking |
+        """
+        region = to_region(coordinates)
+        timeout = float(timeout)
+        if timeout > 0:
+            self.ctx.overlay.highlight(region, timeout)
+            time.sleep(timeout)
+        before = (
+            self.ctx.image.grab(region) if float(wait_change) > 0 else None
+        )
+        self._click_at(self._locate(region))
+        if before is not None:
+            changed = self.ctx._poll(
+                lambda: self.ctx.image.grab(region).tobytes()
+                != before.tobytes(),
+                float(wait_change),
+            )
+            if not changed:
+                raise AssertionError(
+                    f'{region} did not change in {wait_change} seconds.'
+                )
+
+    # ---- drag and drop --------------------------------------------------------
+    def _drag(self, sx, sy, tx, ty):
+        self._move(sx, sy)
+        pyautogui.mouseDown(sx, sy)
+        time.sleep(0.1)
+        pyautogui.moveTo(tx, ty, duration=max(0.3, self.ctx.move_mouse_delay))
+        time.sleep(0.1)
+        pyautogui.mouseUp(tx, ty)
+
+    @keyword
+    @busy_action
+    def drag_and_drop(self, source_element, target_element):
+        """Drags ``source_element`` and drops it on ``target_element``.
+
+        If ``source_element`` is empty, the last match is dragged.
+
+        Examples:
+        | Drag And Drop | file.png | trash.png |
+        | Drag And Drop | name:Item 1 | name:Drop Area |
+        | Drag And Drop | ${EMPTY} | trash.png |
+        """
+        src = self.ctx.resolve_or_last(source_element)
+        dst = self._locate(target_element)
+        self._drag(*src.target, *dst.target)
+
+    @keyword
+    @busy_action
+    def drag_and_drop_by_offset(
+        self, source_element, x_offset: int, y_offset: int
+    ):
+        """Drags ``source_element`` and drops it ``x_offset``/``y_offset`` pixels away.
+
+        If ``source_element`` is empty, the last match is dragged.
+
+        Examples:
+        | Drag And Drop By Offset | slider.png | 100 | 0 |
+        | Drag And Drop By Offset | ${EMPTY} | 0 | -50 |
+        """
+        src = self.ctx.resolve_or_last(source_element)
+        sx, sy = src.target
+        self._drag(sx, sy, sx + int(x_offset), sy + int(y_offset))
+
+    # ---- move / buttons / wheel -------------------------------------------------
+    @keyword
+    def mouse_move(self, element=''):
+        """Moves the mouse pointer to ``element``.
+
+        If ``element`` is empty, moves to the last match.
+
+        Examples:
+        | Mouse Move | menu.png |
+        | Mouse Move | automation_id:btnHelp |
+        | Mouse Move | |
+        """
+        match = self.ctx.resolve_or_last(element)
+        self._move(*match.target)
+
+    @keyword
+    def mouse_move_location(self, x: int = 0, y: int = 0):
+        """Moves the mouse pointer to the absolute screen coordinate ``x``, ``y``.
+
+        Example:
+        | Mouse Move Location | 20 | 20 |
+        """
+        self._move(int(x), int(y))
+
+    @keyword
+    def mouse_move_region(self, coordinates, highlight_timeout: float = 0):
+        """Moves the mouse pointer to the center of region ``coordinates``.
+
+        Optionally highlights the region for ``highlight_timeout`` seconds.
+
+        Example:
+        | Mouse Move Region | [20, 20, 20, 20] |
+        | Mouse Move Region | ${coordinates} | 2 |
+        """
+        region = to_region(coordinates)
+        if float(highlight_timeout) > 0:
+            self.ctx.overlay.highlight(region, float(highlight_timeout))
+        self._move(*region.center)
+
+    @keyword
+    @busy_action
+    def mouse_down(self, *mouse_buttons):
+        """Presses and holds the given mouse buttons (``LEFT``, ``MIDDLE``, ``RIGHT``).
+
+        Defaults to ``LEFT`` when no button is given.
+
+        Example:
+        | Mouse Move | item.png |
+        | Mouse Down | LEFT |
+        | Mouse Move | target.png |
+        | Mouse Up |
+        """
+        self._ready()
+        for b in mouse_buttons or ('LEFT',):
+            btn = _BUTTONS[str(b).upper()]
+            pyautogui.mouseDown(button=btn)
+            self._held.add(btn)
+
+    @keyword
+    @busy_action
+    def mouse_up(self, *mouse_buttons):
+        """Releases the given mouse buttons. If empty, releases all held buttons.
+
+        Example:
+        | Mouse Up | LEFT | RIGHT |
+        | Mouse Up |
+        """
+        buttons = (
+            [_BUTTONS[str(b).upper()] for b in mouse_buttons]
+            or list(self._held)
+            or ['left']
+        )
+        for btn in buttons:
+            pyautogui.mouseUp(button=btn)
+            self._held.discard(btn)
+
+    def _wheel(self, steps, element, direction):
+        if element not in (None, ''):
+            self._move(*self._locate(element).target)
+        self._ready()
+        for _ in range(int(steps)):
+            pyautogui.scroll(direction * 120)
+            time.sleep(0.02)
+
+    @keyword
+    @busy_action
+    def wheel_down(self, steps: int, element=''):
+        """Moves the mouse to ``element`` (optional) and scrolls down ``steps`` notches.
+
+        Examples:
+        | Wheel Down | 5 |
+        | Wheel Down | 5 | list.png |
+        """
+        self._wheel(steps, element, -1)
+
+    @keyword
+    @busy_action
+    def wheel_up(self, steps: int, element=''):
+        """Moves the mouse to ``element`` (optional) and scrolls up ``steps`` notches.
+
+        Examples:
+        | Wheel Up | 5 |
+        | Wheel Up | 5 | automation_id:lstItems |
+        """
+        self._wheel(steps, element, 1)

DesktopLibrary/keywords/ocr.py

@@ -0,0 +1,166 @@
+import unicodedata
+
+from robot.api.deco import keyword
+
+from ..geometry import Match, Region, to_region
+from .base import KeywordGroup, busy_action
+
+_ENGINE = None
+
+
+def _engine():
+    global _ENGINE
+    if _ENGINE is None:
+        try:
+            from rapidocr import RapidOCR
+        except ImportError as err:
+            raise RuntimeError(
+                "OCR requires 'rapidocr' and 'onnxruntime': "
+                'pip install robotframework-desktop[ocr]'
+            ) from err
+        _ENGINE = RapidOCR()
+    return _ENGINE
+
+
+def _normalize(text: str) -> str:
+    decomposed = unicodedata.normalize('NFKD', str(text))
+    plain = ''.join(c for c in decomposed if not unicodedata.combining(c))
+    return ' '.join(plain.lower().split())
+
+
+def _ocr_lines(image):
+    """Runs OCR and returns ``[(text, x1, y1, x2, y2), ...]`` in reading order."""
+    import cv2
+    import numpy as np
+
+    array = cv2.cvtColor(np.array(image.convert('RGB')), cv2.COLOR_RGB2BGR)
+    result = _engine()(array)
+    boxes = getattr(result, 'boxes', None)
+    texts = getattr(result, 'txts', None)
+    if boxes is None or texts is None:
+        return []
+    lines = []
+    for box, text in zip(boxes, texts):
+        if not str(text).strip():
+            continue
+        xs = [float(p[0]) for p in box]
+        ys = [float(p[1]) for p in box]
+        lines.append((str(text).strip(), min(xs), min(ys), max(xs), max(ys)))
+    if not lines:
+        return []
+    height = sorted(l[4] - l[2] for l in lines)[len(lines) // 2] or 1
+    return sorted(lines, key=lambda l: (round(l[2] / (height * 0.6)), l[1]))
+
+
+class OcrKeywords(KeywordGroup):
+    def _check(self):
+        if not self.ctx.ocr_enabled:
+            raise RuntimeError(
+                "OCR is disabled. Use 'Set Ocr Text Read    true'."
+            )
+
+    def _read(self, region: Region) -> str:
+        self._check()
+        lines = _ocr_lines(self.ctx.image.grab(region))
+        rows, last_y, height = [], None, None
+        for text, x1, y1, x2, y2 in lines:
+            if last_y is not None and abs(y1 - last_y) <= (height or 0) * 0.6:
+                rows[-1].append(text)
+            else:
+                rows.append([text])
+                last_y, height = y1, y2 - y1
+        return '\n'.join(' '.join(row) for row in rows).strip()
+
+    def _find_text(self, text: str, region: Region) -> Match:
+        self._check()
+        wanted = _normalize(text)
+        if not wanted:
+            raise ValueError('Text cannot be empty.')
+
+        def search():
+            for line, x1, y1, x2, y2 in _ocr_lines(
+                self.ctx.image.grab(region)
+            ):
+                normalized = _normalize(line)
+                start = normalized.find(wanted)
+                if start < 0:
+                    continue
+                char_w = (x2 - x1) / max(len(normalized), 1)
+                left = x1 + start * char_w
+                width = max(len(wanted) * char_w, 1)
+                return Match(
+                    int(region.x + left),
+                    int(region.y + y1),
+                    int(round(width)),
+                    int(round(y2 - y1)),
+                    source=f'text:{text}',
+                )
+            return None
+
+        match = self.ctx._poll(search, self.ctx.timeout)
+        if not match:
+            raise AssertionError(f"Text '{text}' not found on screen.")
+        self.ctx.last_match = match
+        return match
+
+    @keyword
+    def set_ocr_text_read(self, ocr_text_read: bool):
+        """Enables or disables OCR keywords (enabled by default).
+
+        Example:
+        | Set Ocr Text Read | true |
+        """
+        self.ctx.ocr_enabled = str(ocr_text_read).lower() in (
+            'true',
+            '1',
+            'yes',
+            'on',
+        )
+
+    @keyword
+    def get_text(self, element='') -> str:
+        """Reads text with OCR (RapidOCR) from ``element`` area or from the
+        whole screen.
+
+        Examples:
+        | ${all}= | Get Text |
+        | ${txt}= | Get Text | status_area.png |
+        | ${txt}= | Get Text | automation_id:lblStatus |
+        """
+        if element in (None, ''):
+            return self._read(self.ctx.search_region())
+        match = self.ctx.find(element)
+        return self._read(Region(*match.to_tuple()))
+
+    @keyword
+    def read_text_from_region(self, region) -> str:
+        """Reads text with OCR from ``region`` ``[x, y, w, h]``.
+
+        Example:
+        | ${region}= | Get Extended Region From Image | label.png | right | 3 |
+        | ${text}= | Read Text From Region | ${region} |
+        """
+        return self._read(to_region(region))
+
+    @keyword
+    @busy_action
+    def click_text(self, text: str):
+        """Clicks ``text`` found with OCR on the current screen. Matching is
+        case and accent insensitive.
+
+        Example:
+        | Click Text | Hello |
+        """
+        self._click_at(self._find_text(text, self.ctx.screen()))
+
+    @keyword
+    @busy_action
+    def region_click_text(self, text: str):
+        """Clicks ``text`` found with OCR inside the region of interest
+        (see `Set ROI`).
+
+        Example:
+        | Set ROI | [0, 0, 600, 400] |
+        | Region Click Text | Hello |
+        """
+        self._click_at(self._find_text(text, self.ctx.search_region()))