robotframework-desktop 0.1.0__py3-none-any.whl

DesktopLibrary/__init__.py

@@ -0,0 +1,278 @@
+import sys
+
+if (
+    sys.platform == 'win32'
+):  # make screen, mouse and UIA coordinates consistent
+    import ctypes
+
+    try:
+        ctypes.windll.shcore.SetProcessDpiAwareness(2)
+    except Exception:
+        try:
+            ctypes.windll.user32.SetProcessDPIAware()
+        except Exception:
+            pass
+
+import pyautogui  # noqa: E402
+from robotlibcore import DynamicCore  # noqa: E402
+
+from .context import ElementNotFound, LibraryContext  # noqa: E402
+from .geometry import Match, Region  # noqa: E402
+from .keywords import (  # noqa: E402
+    ApplicationKeywords,
+    ConfigKeywords,
+    CursorKeywords,
+    ElementKeywords,
+    FindKeywords,
+    KeyboardKeywords,
+    MouseKeywords,
+    OcrKeywords,
+    RegionKeywords,
+    WaitKeywords,
+)
+from .locator import Locator, parse_locator  # noqa: E402
+
+__version__ = '0.1.0'
+__all__ = [
+    'DesktopLibrary',
+    'ElementNotFound',
+    'Locator',
+    'Match',
+    'Region',
+    'parse_locator',
+]
+
+pyautogui.PAUSE = 0.05
+
+
+class DesktopLibrary(DynamicCore):
+    """DesktopLibrary is a Robot Framework library for Windows desktop
+    automation. It combines image recognition (``pyautogui`` + OpenCV) with
+    UI Automation (``pywinauto``) for WPF, WinForms, UWP and Win32 applications.
+
+    Keywords are *hybrid*: the engine used is chosen by the format of the
+    ``element`` argument.
+
+    = Element locators =
+
+    == Image file path ==
+
+    A string is treated as an image path unless it matches the UI locator
+    syntax. Relative paths are resolved against the folders given with the
+    ``image_paths`` import argument or `Add Image Path`, in order, then the
+    current working directory. The extension can be omitted
+    (``.png``, ``.jpg``, ``.jpeg`` and ``.bmp`` are tried).
+
+    The default similarity is ``0.6`` and can be changed with
+    `Set Min Similarity`. A per-call similarity can be given with
+    ``image.png=0.85``. Use the ``image:`` prefix to force the image engine.
+
+    | Click | buttons/save.png |
+    | Click | save.png=0.9 |
+    | Click | image:C:/refs/save.png |
+
+    == Dictionary ==
+
+    | =Key= | =Description= |
+    | path | Image path (string) or list of image paths. With a list, the element is *any* of the images (the best match wins). |
+    | similarity | Similarity for this locator only (0-1). |
+    | offset | Dictionary ``{"x": int, "y": int}`` added to the center of the element when acting. |
+    | region | Dictionary ``{"x", "y", "width", "height"}`` limiting where the element is searched. |
+    | name, automation_id, class_name, control_type, ... | UI Automation criteria (instead of ``path``). |
+    | backend | ``uia`` or ``win32`` for UI criteria. |
+    | locator | Any other locator string, so ``offset``/``region`` can be added to it. |
+
+    | ${ok}= | Create Dictionary | path=ok.png | similarity=0.85 |
+    | Click | ${ok} |
+    | Click | ${{ {"path": ["ok.png", "ok_dark.png"], "offset": {"x": 10, "y": 0}} }} |
+    | Click | ${{ {"path": "ok.png", "region": {"x": 0, "y": 0, "width": 800, "height": 600}} }} |
+    | Click | ${{ {"automation_id": "btnSave", "offset": {"x": 5, "y": 5}} }} |
+
+    == UI Automation locator ==
+
+    Strings in the form ``key:value`` use pywinauto. Conditions can be combined
+    with ``AND``. A ``uia:`` or ``win32:`` prefix selects the backend
+    (default is the ``backend`` import argument, ``uia``).
+
+    | =Key= | =Meaning= |
+    | name / title | Element name (window title, button text...). |
+    | name_re / title_re | Regular expression for the name. |
+    | automation_id / id / auto_id | AutomationId (WPF ``x:Name``, WinForms ``Name``). |
+    | class / class_name | Class name. |
+    | control_type / type | Control type: ``Button``, ``Edit``, ``ComboBox``... (uia only). |
+    | index / found_index | 0 based index when multiple elements match. |
+    | handle, process | Window handle or process id. |
+
+    | Attach To Window | name:Sample App |
+    | Click | automation_id:btnSave |
+    | Click | name:OK AND control_type:Button |
+    | Click | win32:class:Button AND name:Cancel |
+
+    Searches are made inside the window set with `Attach To Window`, or in the
+    whole desktop otherwise (slower).
+
+    == Region and match ==
+
+    Regions (``[x, y, width, height]`` or values returned by keywords such
+    as `Get Extended Region From Image`) and matches can be used as elements:
+    the action happens at their center.
+
+    == Looping over elements ==
+
+    `Find Element`, `Find Elements` and `Find Child Elements` return matches
+    that can be passed to any keyword. UI matches keep the UI Automation
+    element; image matches are screen positions (they become stale if the
+    screen changes).
+
+    | @{items}= | Find Elements | control_type:ListItem |
+    | FOR | ${item} | IN | @{items} |
+    | | ${text}= | Get Element Text | ${item} |
+    | | IF | $text == "Item 3" | Click | ${item} |
+    | END |
+
+    = Importing =
+
+    | Library | DesktopLibrary | image_paths=${CURDIR}/images;${CURDIR}/shared | similarity=0.7 | timeout=10 |
+
+    = Busy cursor handling =
+
+    Desktop applications often load data in background, so a fixed wait is
+    not reliable. DesktopLibrary watches the mouse cursor (Windows
+    ``GetCursorInfo``) and, when it is *busy* (``wait`` or ``appstarting`` by
+    default), waits until it is not busy for a stability window
+    (``0.5`` seconds by default):
+
+    - before searching elements in action keywords (`Click`, `Input Text`, `Drag And Drop`,
+      `Get Element Text`, `Wait Until Screen Contain`, ...),
+    - before mouse and keyboard actions, also without element (`Press Special Key`,
+      `Type With Modifiers`, `Input Text` with ``${EMPTY}``, `Mouse Down`, wheel...),
+    - after action keywords: the cursor is watched for ``busy_cursor_after_delay``
+      (``0.2s`` by default) and, if the action made it busy, the keyword only
+      returns when the application is ready again.
+
+    Action keywords accept the named argument ``ignore_busy_error=True`` to log
+    a warning instead of failing when the busy cursor wait times out:
+
+    | Click | automation_id:btnLoad | ignore_busy_error=True |
+
+    Checks such as `Exists`, `Screen Should Not Contain` and `Get Match Score`
+    never wait. `Wait Until Screen Contain`/`Screen Should Contain` limit the
+    cursor wait to their own timeout. The cursor is read at its current
+    position; the mouse is not moved for the check.
+
+    If the cursor stays busy longer than ``busy_cursor_timeout`` (default
+    ``30s``) the keyword fails by default, or logs a warning and continues when
+    ``busy_cursor_fail=False``. Application specific cursors are reported as
+    ``custom`` and are *not* busy unless added with `Set Busy Cursor Types`.
+
+    | Library | DesktopLibrary | busy_cursor_timeout=1 min | busy_cursor_fail=False |
+    | Set Busy Cursor Wait | False | | | # disable automatic waiting |
+    | Set Busy Cursor Types | wait,appstarting,custom |
+    | Wait Until Cursor Is Not Busy | 2 min |
+    | ${cursor}= | Get Current Cursor Type |
+
+    = Example =
+
+    | *** Settings ***
+    | Library    DesktopLibrary    image_paths=${CURDIR}/images
+    |
+    | *** Test Cases ***
+    | Save Document
+    |     Open Application    notepad.exe
+    |     Attach To Window    name_re:.*Notepad
+    |     Input Text    class:Edit    Hello from Robot
+    |     Type With Modifiers    s    CTRL
+    |     Wait Until Screen Contain    save_dialog.png    10
+    |     Click    ${{ {"path": ["save_btn.png", "save_btn_dark.png"], "similarity": 0.8} }}
+    |     [Teardown]    Close Application    notepad.exe
+    """
+
+    ROBOT_LIBRARY_SCOPE = 'GLOBAL'
+    ROBOT_LIBRARY_VERSION = __version__
+    ROBOT_LIBRARY_DOC_FORMAT = 'ROBOT'
+
+    def __init__(
+        self,
+        image_paths=None,
+        similarity: float = 0.6,
+        timeout='5s',
+        backend: str = 'uia',
+        capture_folder: str | None = None,
+        scan_rate: float = 10.0,
+        screen_id: int = 0,
+        show_actions: bool = False,
+        busy_cursor_wait: bool = True,
+        busy_cursor_timeout='30s',
+        busy_cursor_stable='0.5s',
+        busy_cursor_types='wait,appstarting',
+        busy_cursor_fail: bool = True,
+        busy_cursor_after_delay='0.2s',
+        show_warnings: bool = True,
+    ):
+        """
+        | =Argument= | =Description= |
+        | image_paths | Folders for relative image paths: list or string separated by ``;``. |
+        | similarity | Default image similarity (0-1). Default ``0.6``. |
+        | timeout | Default wait timeout. Default ``5s``. |
+        | backend | Default UI backend: ``uia`` or ``win32``. |
+        | capture_folder | Folder for captures. Default ``${OUTPUT DIR}``. |
+        | scan_rate | Searches per second while waiting. Default ``10``. |
+        | screen_id | Monitor used as default search area: ``0`` primary, ``-1`` all monitors. Default ``0``. |
+        | show_actions | Highlight targets before acting. Default ``False``. |
+        | busy_cursor_wait | Automatically wait while the cursor is busy. Default ``True``. |
+        | busy_cursor_timeout | Maximum busy cursor wait. Default ``30s``. |
+        | busy_cursor_stable | Time the cursor must stay not busy. Default ``0.5s``. |
+        | busy_cursor_types | Cursor types considered busy. Default ``wait,appstarting``. |
+        | busy_cursor_fail | Fail (``True``) or warn and continue (``False``) on busy timeout. Default ``True``. |
+        | busy_cursor_after_delay | Time watched after an action for the cursor to become busy. Default ``0.2s``. |
+        | show_warnings | Emit library warnings (e.g. ignored busy cursor timeouts). When ``False`` they are logged as INFO only. Default ``True``. |
+
+        Examples:
+        | Library | DesktopLibrary |
+        | Library | DesktopLibrary | image_paths=${CURDIR}/images | similarity=0.8 |
+        | Library | DesktopLibrary | backend=win32 | timeout=10s |
+        | Library | DesktopLibrary | busy_cursor_timeout=2 min | busy_cursor_fail=False |
+        | Library | DesktopLibrary | busy_cursor_fail=False | show_warnings=False |
+        """
+
+        def as_bool(value):
+            return (
+                value
+                if isinstance(value, bool)
+                else str(value).lower() in ('true', '1', 'yes', 'on')
+            )
+
+        show_actions = as_bool(show_actions)
+        self.ctx = LibraryContext(
+            image_paths=image_paths,
+            similarity=similarity,
+            timeout=timeout,
+            backend=backend,
+            capture_folder=capture_folder,
+            scan_rate=scan_rate,
+            screen_id=screen_id,
+            show_actions=show_actions,
+            busy_cursor_wait=as_bool(busy_cursor_wait),
+            busy_cursor_timeout=busy_cursor_timeout,
+            busy_cursor_stable=busy_cursor_stable,
+            busy_cursor_types=busy_cursor_types,
+            busy_cursor_fail=as_bool(busy_cursor_fail),
+            busy_cursor_after_delay=busy_cursor_after_delay,
+            show_warnings=as_bool(show_warnings),
+        )
+        components = [
+            cls(self.ctx)
+            for cls in (
+                MouseKeywords,
+                KeyboardKeywords,
+                WaitKeywords,
+                RegionKeywords,
+                OcrKeywords,
+                ApplicationKeywords,
+                ElementKeywords,
+                ConfigKeywords,
+                CursorKeywords,
+                FindKeywords,
+            )
+        ]
+        super().__init__(components)

DesktopLibrary/context.py

@@ -0,0 +1,272 @@
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+import mss
+from robot.api import logger
+from robot.utils import timestr_to_secs
+
+from .cursor import CursorMonitor, normalize_cursor_types
+from .engines import ImageEngine, ImagePathResolver, UIEngine
+from .geometry import Match, Region
+from .locator import Locator, parse_locator
+from .overlay import Overlay
+
+
+class ElementNotFound(AssertionError):
+    pass
+
+
+class LibraryContext:
+    """Shared state and core lookup logic used by every keyword group."""
+
+    def __init__(
+        self,
+        image_paths=None,
+        similarity=0.6,
+        timeout=5.0,
+        backend='uia',
+        capture_folder=None,
+        scan_rate=10.0,
+        screen_id=0,
+        show_actions=False,
+        busy_cursor_wait=True,
+        busy_cursor_timeout=30.0,
+        busy_cursor_stable=0.5,
+        busy_cursor_types='wait,appstarting',
+        busy_cursor_fail=True,
+        busy_cursor_after_delay=0.2,
+        show_warnings=True,
+    ):
+        self.show_warnings = show_warnings
+        self.cursor = CursorMonitor(
+            enabled=busy_cursor_wait,
+            timeout=self.to_seconds(busy_cursor_timeout),
+            stable_duration=self.to_seconds(busy_cursor_stable),
+            busy_types=normalize_cursor_types(busy_cursor_types),
+            fail_on_timeout=busy_cursor_fail,
+            after_delay=self.to_seconds(busy_cursor_after_delay),
+            logger=self.log,
+        )
+        self.resolver = ImagePathResolver()
+        for p in self._split_paths(image_paths):
+            self.resolver.add(p)
+        self.image = ImageEngine(self.resolver)
+        self.ui = UIEngine(backend=str(backend).lower())
+        self.overlay = Overlay()
+        self.similarity = float(similarity)
+        self.timeout = self.to_seconds(timeout)
+        self.scan_rate = float(scan_rate)
+        self.screen_id = int(screen_id)
+        self.roi: Region | None = None
+        self.last_match: Match | None = None
+        self._capture_folder = capture_folder
+        self.capture_matched_image = False
+        self.show_actions = bool(show_actions)
+        self.slow_motion_delay = 0.5
+        self.move_mouse_delay = 0.0
+        self.ocr_enabled = True
+        self._capture_counter = 0
+
+    # ---- helpers ------------------------------------------------------------
+    def log(self, message: str, level: str = 'INFO'):
+        """Logs ``message``; warnings are downgraded to INFO when
+        ``show_warnings`` is disabled (kept in the log, hidden from console/report)."""
+        if level.upper() == 'WARN' and not self.show_warnings:
+            level = 'INFO'
+            message = f'[warning suppressed] {message}'
+        logger.write(message, level)
+
+    @staticmethod
+    def _split_paths(value) -> list[str]:
+        if not value:
+            return []
+        if isinstance(value, str):
+            return [p.strip() for p in value.split(';') if p.strip()]
+        return [str(p) for p in value]
+
+    @staticmethod
+    def to_seconds(value, default: float | None = None) -> float:
+        if value is None or (isinstance(value, str) and value.strip() == ''):
+            if default is None:
+                raise ValueError('Timeout value is required.')
+            return default
+        return float(timestr_to_secs(value))
+
+    @property
+    def capture_folder(self) -> Path:
+        folder = self._capture_folder
+        if not folder:
+            try:
+                from robot.libraries.BuiltIn import BuiltIn
+
+                folder = BuiltIn().get_variable_value('${OUTPUT DIR}')
+            except Exception:
+                folder = None
+        path = Path(folder or Path.cwd())
+        path.mkdir(parents=True, exist_ok=True)
+        return path
+
+    @capture_folder.setter
+    def capture_folder(self, value):
+        self._capture_folder = value
+
+    def next_capture_path(
+        self, name: str = '', prefix: str = 'capture'
+    ) -> Path:
+        if name:
+            p = Path(name)
+            if p.suffix.lower() not in ('.png', '.jpg', '.jpeg', '.bmp'):
+                p = p.with_name(p.name + '.png')
+            return p if p.is_absolute() else self.capture_folder / p
+        self._capture_counter += 1
+        stamp = time.strftime('%Y%m%d-%H%M%S')
+        return (
+            self.capture_folder
+            / f'{prefix}-{stamp}-{self._capture_counter}.png'
+        )
+
+    # ---- screens ------------------------------------------------------------
+    @staticmethod
+    def monitors() -> list[Region]:
+        """Monitors with the primary one (containing 0,0) first."""
+        with mss.mss() as sct:
+            mons = sct.monitors[1:] or sct.monitors[:1]
+            regions = [
+                Region(m['left'], m['top'], m['width'], m['height'])
+                for m in mons
+            ]
+        regions.sort(key=lambda r: (not r.contains_point(0, 0), r.x, r.y))
+        return regions
+
+    @staticmethod
+    def virtual_screen() -> Region:
+        with mss.mss() as sct:
+            m = sct.monitors[0]
+            return Region(m['left'], m['top'], m['width'], m['height'])
+
+    def screen(self) -> Region:
+        if self.screen_id == -1:
+            return self.virtual_screen()
+        mons = self.monitors()
+        if not 0 <= self.screen_id < len(mons):
+            raise ValueError(
+                f'Screen id {self.screen_id} out of range (0..{len(mons) - 1}).'
+            )
+        return mons[self.screen_id]
+
+    def search_region(self, locator: Locator | None = None) -> Region:
+        if locator is not None and locator.region is not None:
+            return locator.region
+        return self.roi or self.screen()
+
+    # ---- lookup ---------------------------------------------------------------
+    def find_now(self, locator: Locator) -> list[Match]:
+        if locator.kind == 'region':
+            fixed = locator.fixed
+            return [
+                Match(
+                    fixed.x,
+                    fixed.y,
+                    fixed.width,
+                    fixed.height,
+                    score=getattr(fixed, 'score', 1.0),
+                    source='region',
+                    offset=tuple(locator.offset),
+                    element=getattr(fixed, 'element', None),
+                )
+            ]
+        if locator.kind == 'image':
+            similarity = locator.similarity or self.similarity
+            return self.image.find_all(
+                locator, self.search_region(locator), similarity
+            )
+        return self.ui.find_all(locator)
+
+    def _poll(self, condition: Callable[[], Any], timeout: float):
+        deadline = time.monotonic() + timeout
+        interval = 1.0 / self.scan_rate if self.scan_rate > 0 else 0.1
+        while True:
+            result = condition()
+            if result:
+                return result
+            if time.monotonic() >= deadline:
+                return result
+            time.sleep(interval)
+
+    def find(
+        self,
+        target: Any,
+        timeout: Any = None,
+        region: Region | None = None,
+        x_offset: int = 0,
+        y_offset: int = 0,
+        wait_cursor: bool = True,
+    ) -> Match:
+        locator = parse_locator(target)
+        if region is not None:
+            locator = locator.with_region(region)
+        if x_offset or y_offset:
+            locator = locator.with_extra_offset(x_offset, y_offset)
+        secs = self.to_seconds(timeout, self.timeout)
+        if wait_cursor:
+            self.cursor.auto_wait()
+        matches = self._poll(lambda: self.find_now(locator), secs)
+        if not matches:
+            raise ElementNotFound(
+                f"Element '{locator.describe()}' not found in {secs:g} seconds."
+            )
+        match = max(matches, key=lambda m: m.score)
+        self.last_match = match
+        self._after_match(match)
+        return match
+
+    def find_all(
+        self, target: Any, timeout: Any = 0, wait_cursor: bool = False
+    ) -> list[Match]:
+        locator = parse_locator(target)
+        secs = self.to_seconds(timeout, 0)
+        if wait_cursor:
+            self.cursor.auto_wait()
+        return self._poll(lambda: self.find_now(locator), secs) or []
+
+    def exists(self, target: Any, timeout: Any = None) -> bool:
+        try:
+            self.find(target, timeout, wait_cursor=False)
+            return True
+        except ElementNotFound:
+            return False
+
+    def wait_vanish(self, target: Any, timeout: Any = None) -> bool:
+        locator = parse_locator(target)
+        secs = self.to_seconds(timeout, self.timeout)
+        self.cursor.auto_wait(secs)
+        return bool(self._poll(lambda: not self.find_now(locator), secs))
+
+    def resolve_or_last(self, target: Any, timeout: Any = None) -> Match:
+        if target is None or (
+            isinstance(target, str) and target.strip() == ''
+        ):
+            if self.last_match is None:
+                raise ElementNotFound('No previous match available.')
+            return self.last_match
+        return self.find(target, timeout)
+
+    # ---- visual feedback ----------------------------------------------------
+    def _after_match(self, match: Match):
+        if self.capture_matched_image:
+            try:
+                path = self.next_capture_path(prefix='match')
+                self.image.grab(match).save(path)
+                logger.info(
+                    f'<img src="{path.as_posix()}" width="{min(match.width, 400)}">',
+                    html=True,
+                )
+            except Exception as err:  # pragma: no cover - diagnostics only
+                logger.debug(f'Could not capture matched image: {err}')
+        logger.info(f'Found {match}')
+
+    def show_action(self, region: Region):
+        if self.show_actions:
+            self.overlay.highlight(region, self.slow_motion_delay)
+            time.sleep(self.slow_motion_delay)