robotframework-desktop 0.1.0__py3-none-any.whl

DesktopLibrary/cursor.py

@@ -0,0 +1,243 @@
+"""Mouse cursor inspection (Windows ``GetCursorInfo``) and busy-cursor waiting."""
+
+import ctypes
+import sys
+import time
+from contextlib import contextmanager
+from ctypes import wintypes
+
+CURSOR_IDS = {
+    'arrow': 32512,
+    'ibeam': 32513,
+    'wait': 32514,
+    'cross': 32515,
+    'uparrow': 32516,
+    'sizenwse': 32642,
+    'sizenesw': 32643,
+    'sizewe': 32644,
+    'sizens': 32645,
+    'sizeall': 32646,
+    'no': 32648,
+    'hand': 32649,
+    'appstarting': 32650,
+    'help': 32651,
+}
+ID_TO_NAME = {v: k for k, v in CURSOR_IDS.items()}
+SPECIAL_TYPES = ('custom', 'none')
+_CURSOR_SHOWING = 0x00000001
+
+
+class BusyCursorTimeout(AssertionError):
+    pass
+
+
+class _POINT(ctypes.Structure):
+    _fields_ = [('x', wintypes.LONG), ('y', wintypes.LONG)]
+
+
+class _CURSORINFO(ctypes.Structure):
+    _fields_ = [
+        ('cbSize', wintypes.DWORD),
+        ('flags', wintypes.DWORD),
+        ('hCursor', ctypes.c_void_p),
+        ('ptScreenPos', _POINT),
+    ]
+
+
+def normalize_cursor_types(value) -> list[str]:
+    """Accepts a name, an id, a comma separated string or a list."""
+    if isinstance(value, (list, tuple, set)):
+        items = list(value)
+    elif isinstance(value, str):
+        items = [v for v in value.split(',')]
+    else:
+        items = [value]
+    result = []
+    for item in items:
+        text = str(item).strip().lower()
+        if not text:
+            continue
+        if text.isdigit() and int(text) in ID_TO_NAME:
+            text = ID_TO_NAME[int(text)]
+        if text not in CURSOR_IDS and text not in SPECIAL_TYPES:
+            raise ValueError(
+                f"Invalid cursor type '{item}'. Valid: "
+                f"{', '.join([*CURSOR_IDS, *SPECIAL_TYPES])} or their numeric ids."
+            )
+        result.append(text)
+    if not result:
+        raise ValueError('At least one cursor type is required.')
+    return result
+
+
+class CursorMonitor:
+    """Reads the current cursor type and waits for it to stop being busy."""
+
+    def __init__(
+        self,
+        enabled=True,
+        timeout=30.0,
+        stable_duration=0.5,
+        busy_types=('wait', 'appstarting'),
+        fail_on_timeout=True,
+        poll_interval=0.05,
+        reader=None,
+        logger=None,
+        after_delay=0.2,
+    ):
+        self.enabled = enabled
+        self.timeout = float(timeout)
+        self.stable_duration = float(stable_duration)
+        self.busy_types = normalize_cursor_types(list(busy_types))
+        self.fail_on_timeout = fail_on_timeout
+        self.after_delay = float(after_delay)
+        self.poll_interval = poll_interval
+        self._reader = reader
+        self._log = logger or (lambda msg, level='INFO': None)
+        self._handles: dict[int, str] | None = None
+        self._ignore_errors = 0
+
+    @contextmanager
+    def action(self, ignore_errors: bool = False):
+        """Wraps a keyword action: errors become warnings when ``ignore_errors``
+        and the busy cursor is awaited after the action completes."""
+        if ignore_errors:
+            self._ignore_errors += 1
+        try:
+            yield
+            self.after_action()
+        finally:
+            if ignore_errors:
+                self._ignore_errors -= 1
+
+    def after_action(self):
+        """Watches ``after_delay`` seconds for a busy cursor caused by the
+        action and waits for it to finish."""
+        if not self.enabled:
+            return
+        deadline = time.monotonic() + self.after_delay
+        while True:
+            if self.current() in self.busy_types:
+                self._log(
+                    'Busy cursor detected after action, waiting for the application.'
+                )
+                self.wait_not_busy()
+                return
+            if time.monotonic() >= deadline:
+                return
+            time.sleep(self.poll_interval)
+
+    # ---- reading ------------------------------------------------------------
+    def _system_handles(self) -> dict[int, str]:
+        if self._handles is None:
+            user32 = ctypes.WinDLL('user32', use_last_error=True)
+            user32.LoadCursorW.argtypes = (wintypes.HINSTANCE, ctypes.c_void_p)
+            user32.LoadCursorW.restype = ctypes.c_void_p
+            self._handles = {}
+            for name, idc in CURSOR_IDS.items():
+                handle = user32.LoadCursorW(None, ctypes.c_void_p(idc))
+                if handle:
+                    self._handles[int(handle)] = name
+        return self._handles
+
+    def current(self) -> str:
+        """Returns the cursor type name, ``custom`` or ``none``."""
+        if self._reader is not None:
+            return self._reader()
+        if sys.platform != 'win32':
+            return 'none'
+        info = _CURSORINFO()
+        info.cbSize = ctypes.sizeof(_CURSORINFO)
+        user32 = ctypes.WinDLL('user32', use_last_error=True)
+        if not user32.GetCursorInfo(ctypes.byref(info)):
+            raise ctypes.WinError(ctypes.get_last_error())
+        if not info.flags & _CURSOR_SHOWING or not info.hCursor:
+            return 'none'
+        return self._system_handles().get(int(info.hCursor), 'custom')
+
+    def is_busy(self, cursor: str | None = None) -> bool:
+        return (cursor or self.current()) in self.busy_types
+
+    # ---- waiting ------------------------------------------------------------
+    def wait_until(
+        self, accept, timeout: float, stable_duration: float, description: str
+    ) -> tuple[bool, str, list]:
+        """Waits until ``accept(cursor)`` is true continuously for
+        ``stable_duration``. Returns ``(ok, last_cursor, changes)``."""
+        start = time.monotonic()
+        deadline = start + timeout
+        last = None
+        stable_since = None
+        changes = []
+        while True:
+            now = time.monotonic()
+            cursor = self.current()
+            if cursor != last:
+                if last is not None:
+                    changes.append((round(now - start, 2), last, cursor))
+                last = cursor
+                stable_since = now if accept(cursor) else None
+            elif stable_since is None and accept(cursor):
+                stable_since = now
+            if (
+                stable_since is not None
+                and now - stable_since >= stable_duration
+            ):
+                return True, cursor, changes
+            if now >= deadline:
+                return False, cursor, changes
+            time.sleep(self.poll_interval)
+
+    def wait_not_busy(
+        self,
+        timeout: float | None = None,
+        stable_duration: float | None = None,
+        fail: bool | None = None,
+    ) -> str:
+        timeout = self.timeout if timeout is None else float(timeout)
+        stable = (
+            self.stable_duration
+            if stable_duration is None
+            else float(stable_duration)
+        )
+        fail = self.fail_on_timeout if fail is None else fail
+        if self._ignore_errors:
+            fail = False
+        first = self.current()
+        if first not in self.busy_types and stable <= 0:
+            return first
+        ok, cursor, changes = self.wait_until(
+            lambda c: c not in self.busy_types, timeout, stable, 'not busy'
+        )
+        if changes:
+            self._log(
+                f'Cursor changes while waiting: {_format_changes(changes)}'
+            )
+        if ok:
+            return cursor
+        message = (
+            f"Cursor still busy after {timeout:g} seconds (current: '{cursor}', "
+            f"busy types: {', '.join(self.busy_types)}). Changes: {_format_changes(changes)}"
+        )
+        if fail:
+            raise BusyCursorTimeout(message)
+        self._log(message, 'WARN')
+        return cursor
+
+    def auto_wait(self, timeout: float | None = None):
+        """Hook called by keywords before acting. No-op when disabled."""
+        if not self.enabled:
+            return
+        if timeout is not None:
+            timeout = min(timeout, self.timeout)
+        if self.current() not in self.busy_types:
+            # Quick path: only wait the stability window if the cursor is busy now.
+            return
+        self._log('Busy cursor detected, waiting for the application.')
+        self.wait_not_busy(timeout)
+
+
+def _format_changes(changes) -> str:
+    if not changes:
+        return 'none'
+    return ', '.join(f'{a}->{b} (t={t}s)' for t, a, b in changes)

DesktopLibrary/engines/__init__.py

@@ -0,0 +1,4 @@
+from .image import ImageEngine, ImagePathResolver
+from .ui import UIEngine
+
+__all__ = ['ImageEngine', 'ImagePathResolver', 'UIEngine']

DesktopLibrary/engines/image.py

@@ -0,0 +1,174 @@
+import os
+from pathlib import Path
+
+import cv2
+import mss
+import numpy as np
+import pyautogui
+from PIL import Image
+
+from ..geometry import Match, Region
+from ..locator import IMAGE_EXTENSIONS, Locator
+
+
+class ImagePathResolver:
+    """Resolves relative image paths against the registered image folders."""
+
+    def __init__(self):
+        self.paths: list[Path] = []
+
+    def add(self, path: str) -> None:
+        p = Path(os.path.expandvars(os.path.expanduser(str(path)))).resolve()
+        if p not in self.paths:
+            self.paths.append(p)
+
+    def remove(self, path: str) -> None:
+        p = Path(os.path.expandvars(os.path.expanduser(str(path)))).resolve()
+        if p in self.paths:
+            self.paths.remove(p)
+
+    def candidates(self, name: str) -> list[Path]:
+        raw = Path(os.path.expandvars(os.path.expanduser(name)))
+        names = [raw]
+        if raw.suffix.lower() not in IMAGE_EXTENSIONS:
+            names += [
+                raw.with_name(raw.name + ext) for ext in IMAGE_EXTENSIONS
+            ]
+        result: list[Path] = []
+        for n in names:
+            if n.is_absolute():
+                result.append(n)
+                continue
+            result += [base / n for base in self.paths]
+            result.append(Path.cwd() / n)
+        return result
+
+    def resolve(self, name: str) -> Path:
+        tried = self.candidates(str(name))
+        for c in tried:
+            if c.is_file():
+                return c.resolve()
+        searched = '\n  '.join(str(t) for t in tried)
+        raise FileNotFoundError(
+            f"Image '{name}' not found. Searched:\n  {searched}"
+        )
+
+
+class ImageEngine:
+    """Image recognition based on pyautogui (OpenCV backed ``confidence``)."""
+
+    def __init__(self, resolver: ImagePathResolver):
+        self.resolver = resolver
+        self.grayscale = False
+        self.resize_factor = 0.0
+        self._cache: dict[tuple[str, float], Image.Image] = {}
+
+    # ---- capture -------------------------------------------------------
+    @staticmethod
+    def grab(region: Region) -> Image.Image:
+        with mss.mss() as sct:
+            shot = sct.grab(
+                {
+                    'left': region.x,
+                    'top': region.y,
+                    'width': region.width,
+                    'height': region.height,
+                }
+            )
+            return Image.frombytes('RGB', shot.size, shot.bgra, 'raw', 'BGRX')
+
+    # ---- needles -------------------------------------------------------
+    def load(self, name: str) -> tuple[Image.Image, Path]:
+        path = self.resolver.resolve(name)
+        key = (str(path), self.resize_factor, os.path.getmtime(path))
+        img = self._cache.get(key)
+        if img is None:
+            img = Image.open(path).convert('RGB')
+            if self.resize_factor and self.resize_factor not in (0, 1):
+                img = img.resize(
+                    (
+                        max(1, int(img.width * self.resize_factor)),
+                        max(1, int(img.height * self.resize_factor)),
+                    )
+                )
+            self._cache[key] = img
+        return img, path
+
+    # ---- search --------------------------------------------------------
+    def find_all(
+        self,
+        locator: Locator,
+        search: Region,
+        similarity: float,
+        haystack: Image.Image | None = None,
+    ) -> list[Match]:
+        if haystack is None:
+            haystack = self.grab(search)
+        matches: list[Match] = []
+        for name in locator.images:
+            needle, path = self.load(name)
+            if (
+                needle.width > haystack.width
+                or needle.height > haystack.height
+            ):
+                continue
+            found: list[Match] = []
+            try:
+                boxes = list(
+                    pyautogui.locateAll(
+                        needle,
+                        haystack,
+                        confidence=similarity,
+                        grayscale=self.grayscale,
+                    )
+                )
+            except Exception as err:
+                if type(err).__name__ != 'ImageNotFoundException':
+                    raise
+                boxes = []
+            for box in boxes:
+                m = Match(
+                    search.x + int(box.left),
+                    search.y + int(box.top),
+                    int(box.width),
+                    int(box.height),
+                    score=0.0,
+                    source=str(path),
+                    offset=tuple(locator.offset),
+                )
+                if any(m.overlap_ratio(o) > 0.5 for o in found):
+                    continue
+                found.append(m)
+            for m in found:
+                crop = haystack.crop(
+                    (
+                        m.x - search.x,
+                        m.y - search.y,
+                        m.right - search.x,
+                        m.bottom - search.y,
+                    )
+                )
+                m.score = self._score(needle, crop)
+            matches.extend(found)
+        return matches
+
+    def best_score(self, locator: Locator, search: Region) -> float:
+        haystack = self.grab(search)
+        best = 0.0
+        for name in locator.images:
+            needle, _ = self.load(name)
+            if (
+                needle.width > haystack.width
+                or needle.height > haystack.height
+            ):
+                continue
+            best = max(best, self._score(needle, haystack))
+        return best
+
+    def _score(self, needle: Image.Image, haystack: Image.Image) -> float:
+        code = cv2.COLOR_RGB2GRAY if self.grayscale else cv2.COLOR_RGB2BGR
+        n = cv2.cvtColor(np.array(needle), code)
+        h = cv2.cvtColor(np.array(haystack), code)
+        result = cv2.matchTemplate(h, n, cv2.TM_CCOEFF_NORMED)
+        _, max_val, _, _ = cv2.minMaxLoc(result)
+        return float(max(0.0, min(1.0, max_val)))

DesktopLibrary/engines/ui.py

@@ -0,0 +1,192 @@
+from typing import Any
+
+from ..geometry import Match, Region
+from ..locator import Locator
+
+
+class UIEngine:
+    """UI Automation / Win32 element lookup based on pywinauto."""
+
+    def __init__(self, backend: str = 'uia'):
+        self.backend = backend
+        self.window = None  # pywinauto wrapper used as search root
+        self.window_backend: str | None = None
+        self.apps: dict[str, Any] = {}
+
+    @staticmethod
+    def _wrap(element_info, backend: str):
+        from pywinauto.backend import registry
+
+        return registry.backends[backend].generic_wrapper_class(element_info)
+
+    def find_elements(
+        self,
+        criteria: dict,
+        backend: str | None = None,
+        parent=None,
+        top_level_only: bool = False,
+        use_window: bool = True,
+    ) -> list:
+        from pywinauto import findwindows
+
+        backend = backend or self.window_backend or self.backend
+        root = (
+            parent
+            if parent is not None
+            else (self.window if use_window else None)
+        )
+        kwargs = dict(criteria)
+        found_index = kwargs.pop('found_index', None)
+        kwargs.setdefault('visible_only', True)
+        if root is not None:
+            kwargs['parent'] = root.element_info
+            kwargs['top_level_only'] = False
+        else:
+            kwargs['top_level_only'] = top_level_only
+        try:
+            infos = findwindows.find_elements(backend=backend, **kwargs)
+        except Exception:
+            infos = []
+        wrappers = [self._wrap(i, backend) for i in infos]
+        if found_index is not None:
+            idx = int(found_index)
+            wrappers = wrappers[idx : idx + 1]
+        return wrappers
+
+    def find_all(self, locator: Locator) -> list[Match]:
+        backend = locator.backend or self.window_backend or self.backend
+        criteria = dict(locator.criteria)
+        if self.window is None:
+            # Cheap first pass: top level windows only.
+            elements = self.find_elements(
+                criteria, backend, top_level_only=True
+            )
+            if not elements:
+                elements = self.find_elements(criteria, backend)
+        else:
+            elements = self.find_elements(criteria, backend)
+            if not elements:
+                # the attached window itself (or another top level window)
+                elements = self.find_elements(
+                    criteria,
+                    backend,
+                    parent=None,
+                    top_level_only=True,
+                    use_window=False,
+                )
+        return self.to_matches(
+            elements, locator.describe(), locator.offset, locator.region
+        )
+
+    def to_matches(
+        self, elements, source: str, offset=(0, 0), region=None
+    ) -> list[Match]:
+        matches = []
+        for el in elements:
+            try:
+                r = el.rectangle()
+            except Exception:
+                continue
+            if r.width() <= 0 or r.height() <= 0:
+                continue
+            m = Match(
+                r.left,
+                r.top,
+                r.width(),
+                r.height(),
+                score=1.0,
+                source=source,
+                offset=tuple(offset),
+                element=el,
+            )
+            if region is not None and not region.contains_point(*m.center):
+                continue
+            matches.append(m)
+        return matches
+
+    def find_children(
+        self,
+        parent,
+        criteria: dict | None = None,
+        depth: int | None = None,
+        backend: str | None = None,
+    ) -> list:
+        """Descendants of ``parent`` (a pywinauto wrapper) matching ``criteria``.
+        ``depth=1`` returns direct children only."""
+        from pywinauto import findwindows
+
+        backend = backend or parent.backend.name
+        kwargs = dict(criteria or {})
+        found_index = kwargs.pop('found_index', None)
+        kwargs.setdefault('visible_only', True)
+        if depth is not None:
+            kwargs['depth'] = int(depth)
+        try:
+            infos = findwindows.find_elements(
+                backend=backend,
+                parent=parent.element_info,
+                top_level_only=False,
+                **kwargs
+            )
+        except Exception:
+            infos = []
+        wrappers = [self._wrap(i, backend) for i in infos]
+        if found_index is not None:
+            wrappers = wrappers[int(found_index) : int(found_index) + 1]
+        return wrappers
+
+    # ---- application / window handling --------------------------------
+    def start(
+        self,
+        alias: str,
+        command: str,
+        backend: str | None = None,
+        timeout: float = 10,
+    ):
+        from pywinauto import Application
+
+        app = Application(backend=backend or self.backend).start(
+            command, timeout=timeout
+        )
+        self.apps[alias] = app
+        return app
+
+    def connect(
+        self,
+        alias: str,
+        backend: str | None = None,
+        timeout: float = 10,
+        **kwargs
+    ):
+        from pywinauto import Application
+
+        app = Application(backend=backend or self.backend).connect(
+            timeout=timeout, **kwargs
+        )
+        self.apps[alias] = app
+        return app
+
+    def attach_window(
+        self, criteria: dict, backend: str | None = None, timeout: float = 10
+    ):
+        from pywinauto import Desktop
+
+        backend = backend or self.backend
+        spec = Desktop(backend=backend).window(**criteria)
+        wrapper = spec.wait('exists', timeout=timeout)
+        self.window = wrapper
+        self.window_backend = backend
+        try:
+            wrapper.set_focus()
+        except Exception:
+            pass
+        return wrapper
+
+    def detach(self):
+        self.window = None
+        self.window_backend = None
+
+    @staticmethod
+    def region_of(wrapper) -> Region:
+        r = wrapper.rectangle()
+        return Region(r.left, r.top, r.width(), r.height())