pyautoscene 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

pyautoscene/scene.py

@@ -1,61 +1,61 @@
-from __future__ import annotations
-
-from typing import Callable, TypedDict
-
-from statemachine import State
-
-from pyautoscene.utils import is_valid_variable_name
-
-from .references import ReferenceElement
-from .screen import Region
-
-
-class ActionInfo(TypedDict):
-    """Type definition for action information in a scene."""
-
-    action: Callable[..., None]
-    transitions_to: Scene | None
-
-
-class Scene(State):
-    """A scene represents a state in the GUI automation state machine."""
-
-    def __init__(
-        self,
-        name: str,
-        elements: list[ReferenceElement] | None = None,
-        initial: bool = False,
-    ):
-        assert is_valid_variable_name(name), (
-            f"Invalid scene name: {name}, must be a valid Python identifier."
-        )
-        super().__init__(name, initial=initial)
-        self.elements = elements or []
-        self.actions: dict[str, ActionInfo] = {}
-
-    def action(self, transitions_to: Scene | None = None):
-        """Decorator to register an action for this scene."""
-
-        def decorator(func: Callable[..., None]) -> Callable[..., None]:
-            if func.__name__ not in self.actions:
-                action_name = func.__name__
-                self.actions[action_name] = {
-                    "action": func,
-                    "transitions_to": transitions_to,
-                }
-            return func
-
-        return decorator
-
-    def get_action(self, action_name: str) -> ActionInfo | None:
-        """Get an action by name."""
-        return self.actions.get(action_name)
-
-    def is_on_screen(self, region: Region | None = None) -> bool:
-        """Check if any reference element is currently on screen."""
-        # TODO: Refactor after text recognition is implemented
-        # elements = (elem for elem in self.elements if isinstance(elem, ReferenceImage))
-        return all(elem.is_visible(region) for elem in self.elements)
-
-    def __repr__(self):
-        return f"Scene({self.name!r}, elements={len(self.elements)})"
+from __future__ import annotations
+
+from typing import Callable, TypedDict
+
+from statemachine import State
+
+from pyautoscene.utils import is_valid_variable_name
+
+from .references import ReferenceElement
+from .region import Region
+
+
+class ActionInfo(TypedDict):
+    """Type definition for action information in a scene."""
+
+    action: Callable[..., None]
+    transitions_to: Scene | None
+
+
+class Scene(State):
+    """A scene represents a state in the GUI automation state machine."""
+
+    def __init__(
+        self,
+        name: str,
+        elements: list[ReferenceElement] | None = None,
+        initial: bool = False,
+    ):
+        assert is_valid_variable_name(name), (
+            f"Invalid scene name: {name}, must be a valid Python identifier."
+        )
+        super().__init__(name, initial=initial)
+        self.elements = elements or []
+        self.actions: dict[str, ActionInfo] = {}
+
+    def action(self, transitions_to: Scene | None = None):
+        """Decorator to register an action for this scene."""
+
+        def decorator(func: Callable[..., None]) -> Callable[..., None]:
+            if func.__name__ not in self.actions:
+                action_name = func.__name__
+                self.actions[action_name] = {
+                    "action": func,
+                    "transitions_to": transitions_to,
+                }
+            return func
+
+        return decorator
+
+    def get_action(self, action_name: str) -> ActionInfo | None:
+        """Get an action by name."""
+        return self.actions.get(action_name)
+
+    def is_on_screen(self, region: Region | None = None) -> bool:
+        """Check if any reference element is currently on screen."""
+        # TODO: Refactor after text recognition is implemented
+        # elements = (elem for elem in self.elements if isinstance(elem, ReferenceImage))
+        return all(elem.locate(region) for elem in self.elements)
+
+    def __repr__(self):
+        return f"Scene({self.name!r}, elements={len(self.elements)})"

pyautoscene/session.py

@@ -1,140 +1,163 @@
-from __future__ import annotations
-
-from typing import Callable
-
-import networkx as nx
-from statemachine import State, StateMachine
-from statemachine.factory import StateMachineMetaclass
-from statemachine.states import States
-from statemachine.transition_list import TransitionList
-
-from .scene import Scene
-from .screen import Region
-
-
-class SceneRecognitionError(Exception):
-    pass
-
-
-def build_dynamic_state_machine(
-    scenes: list[Scene],
-) -> tuple[StateMachine, dict[str, TransitionList], dict[str, Callable]]:
-    """Create a dynamic StateMachine class from scenes using StateMachineMetaclass."""
-
-    states = {scene.name: scene for scene in scenes}
-    transitions = {}
-    leaf_actions = {}
-    for scene in scenes:
-        for action_name, action_info in scene.actions.items():
-            target_scene = action_info["transitions_to"]
-            if target_scene is not None:
-                event_name = f"event_{action_name}"
-                new_transition = scene.to(target_scene, event=event_name)
-                new_transition.on(action_info["action"])
-                transitions[event_name] = new_transition
-            else:
-                leaf_actions[action_name] = action_info["action"]
-
-    SessionSM = StateMachineMetaclass(
-        "SessionSM",
-        (StateMachine,),
-        {"states": States(states), **transitions},  # type: ignore[call-arg]
-    )
-    session_sm: StateMachine = SessionSM()  # type: ignore[no-redef]
-
-    return session_sm, transitions, leaf_actions
-
-
-def get_current_scene(scenes: list[Scene], region: Region | None = None) -> Scene:
-    """Get the current scene from the list of scenes."""
-    current_scenes = [scene for scene in scenes if scene.is_on_screen(region)]
-    if len(current_scenes) == 1:
-        return current_scenes[0]
-    elif len(current_scenes) > 1:
-        raise SceneRecognitionError(
-            f"Multiple scenes are currently on screen.\n{' '.join(str(scene) for scene in current_scenes)}"
-        )
-    else:
-        raise SceneRecognitionError("No scene is currently on screen.")
-
-
-class Session:
-    """A session manages the state machine for GUI automation scenes."""
-
-    def __init__(self, scenes: list[Scene]):
-        self._scenes_list = scenes
-        self._scenes_dict = {scene.name: scene for scene in scenes}
-
-        # Create dynamic StateMachine class and instantiate it
-        self._sm, self.transitions, self.leaf_actions = build_dynamic_state_machine(
-            scenes
-        )
-        self.graph: nx.MultiDiGraph = nx.nx_pydot.from_pydot(self._sm._graph())
-
-    @property
-    def current_scene(self) -> State:
-        """Get the current state."""
-        return self._sm.current_state
-
-    def expect(self, target_scene: Scene, **kwargs):
-        """Navigate to a specific scene."""
-        if target_scene.is_on_screen():
-            return
-
-        present_scene = get_current_scene(self._scenes_list)
-        all_paths = list(
-            nx.all_simple_paths(
-                self.graph,
-                source=present_scene.name,
-                target=target_scene.name,
-            )
-        )
-        if len(all_paths) == 0:
-            raise SceneRecognitionError(
-                f"No path found from {present_scene.name} to {target_scene.name}"
-            )
-        elif len(all_paths) > 1:
-            raise SceneRecognitionError(
-                f"Multiple paths found from {present_scene.name} to {target_scene.name}"
-            )
-
-        path = all_paths[0]
-        events: list[str] = [
-            self.graph.get_edge_data(path[i], path[i + 1])[0]["label"]  # type: ignore
-            for i in range(len(path) - 1)
-        ]
-
-        for event in events:
-            self._sm.send(event, **kwargs)
-
-    def invoke(self, action_name: str, **kwargs):
-        """Invoke an action in the current scene."""
-        event_name = f"event_{action_name}"
-        transition = next(
-            (tr for tr_name, tr in self.transitions.items() if tr_name == event_name),
-            None,
-        )
-        if transition:
-            return self._sm.send(event_name, **kwargs)
-
-        leaf_action = next(
-            (
-                action
-                for name, action in self.leaf_actions.items()
-                if name == action_name
-            ),
-            None,
-        )
-        if leaf_action:
-            return leaf_action(**kwargs)
-
-        raise ValueError(
-            f"Action '{action_name}' not found in current scene '{self.current_scene.name}'"
-        )
-
-    def __repr__(self):
-        current = self.current_scene
-        current_name = current.name if current else "None"
-        return (
-            f"Session(scenes={list(self._scenes_dict.keys())}, current={current_name})"
-        )
+from __future__ import annotations
+
+import time
+from typing import Callable
+
+import networkx as nx
+from PIL import Image
+from statemachine import State, StateMachine
+from statemachine.factory import StateMachineMetaclass
+from statemachine.states import States
+from statemachine.transition_list import TransitionList
+
+from .references import ImageElement, ReferenceElement
+from .region import Region
+from .scene import Scene
+
+
+class SceneRecognitionError(Exception):
+    pass
+
+
+def build_dynamic_state_machine(
+    scenes: list[Scene],
+) -> tuple[StateMachine, dict[str, TransitionList], dict[str, Callable]]:
+    """Create a dynamic StateMachine class from scenes using StateMachineMetaclass."""
+
+    states = {scene.name: scene for scene in scenes}
+    transitions = {}
+    leaf_actions = {}
+    for scene in scenes:
+        for action_name, action_info in scene.actions.items():
+            target_scene = action_info["transitions_to"]
+            if target_scene is not None:
+                event_name = f"event_{action_name}"
+                new_transition = scene.to(target_scene, event=event_name)
+                new_transition.on(action_info["action"])
+                transitions[event_name] = new_transition
+            else:
+                leaf_actions[action_name] = action_info["action"]
+
+    SessionSM = StateMachineMetaclass(
+        "SessionSM",
+        (StateMachine,),
+        {"states": States(states), **transitions},  # type: ignore[call-arg]
+    )
+    session_sm: StateMachine = SessionSM()  # type: ignore[no-redef]
+
+    return session_sm, transitions, leaf_actions
+
+
+def get_current_scene(scenes: list[Scene], region: Region | None = None) -> Scene:
+    """Get the current scene from the list of scenes."""
+    current_scenes = [scene for scene in scenes if scene.is_on_screen(region)]
+    if len(current_scenes) == 1:
+        return current_scenes[0]
+    elif len(current_scenes) > 1:
+        raise SceneRecognitionError(
+            f"Multiple scenes are currently on screen.\n{' '.join(str(scene) for scene in current_scenes)}"
+        )
+    else:
+        raise SceneRecognitionError("No scene is currently on screen.")
+
+
+class Session:
+    """A session manages the state machine for GUI automation scenes."""
+
+    def __init__(
+        self,
+        scenes: list[Scene],
+        image_locator: Callable[[Image.Image, Image.Image], list[Region]] | None = None,
+    ):
+        self._scenes_list = scenes
+        self._scenes_dict = {scene.name: scene for scene in scenes}
+        self.image_locator = image_locator
+        for scene in self._scenes_list:
+            for elem in scene.elements:
+                if isinstance(elem, ImageElement):
+                    elem.locator = image_locator
+
+        # Create dynamic StateMachine class and instantiate it
+        self._sm, self.transitions, self.leaf_actions = build_dynamic_state_machine(
+            scenes
+        )
+        self.graph: nx.MultiDiGraph = nx.nx_pydot.from_pydot(self._sm._graph())
+
+    @property
+    def current_scene(self) -> State:
+        """Get the current state."""
+        return self._sm.current_state
+
+    def expect(self, target_scene: Scene, **kwargs):
+        """Navigate to a specific scene."""
+        if target_scene.is_on_screen():
+            return
+
+        present_scene = get_current_scene(self._scenes_list)
+        all_paths = list(
+            nx.all_simple_paths(
+                self.graph, source=present_scene.name, target=target_scene.name
+            )
+        )
+        if len(all_paths) == 0:
+            raise SceneRecognitionError(
+                f"No path found from {present_scene.name} to {target_scene.name}"
+            )
+        elif len(all_paths) > 1:
+            raise SceneRecognitionError(
+                f"Multiple paths found from {present_scene.name} to {target_scene.name}"
+            )
+
+        path = all_paths[0]
+        events: list[str] = [
+            self.graph.get_edge_data(path[i], path[i + 1])[0]["label"]  # type: ignore
+            for i in range(len(path) - 1)
+        ]
+
+        for event in events:
+            self._sm.send(event, **kwargs)
+
+    def invoke(self, action_name: str, **kwargs):
+        """Invoke an action in the current scene."""
+        event_name = f"event_{action_name}"
+        transition = next(
+            (tr for tr_name, tr in self.transitions.items() if tr_name == event_name),
+            None,
+        )
+        if transition:
+            return self._sm.send(event_name, **kwargs)
+
+        leaf_action = next(
+            (
+                action
+                for name, action in self.leaf_actions.items()
+                if name == action_name
+            ),
+            None,
+        )
+        if leaf_action:
+            return leaf_action(**kwargs)
+
+        raise ValueError(
+            f"Action '{action_name}' not found in current scene '{self.current_scene.name}'"
+        )
+
+    def wait_until(self, target: Scene | ReferenceElement, interval: float = 1):
+        """Wait until the target scene or reference element is on screen."""
+        found = False
+        while not found:
+            if isinstance(target, Scene):
+                found = target.is_on_screen()
+            elif isinstance(target, ReferenceElement):
+                found = target.locate() is not None
+            else:
+                raise TypeError("Target must be a Scene or ReferenceElement.")
+            if not found:
+                time.sleep(interval)
+
+    def __repr__(self):
+        current = self.current_scene
+        current_name = current.name if current else "None"
+        return (
+            f"Session(scenes={list(self._scenes_dict.keys())}, current={current_name})"
+        )

pyautoscene/utils.py

@@ -1,25 +1,148 @@
-from __future__ import annotations
-
-import time
-from keyword import iskeyword
-from typing import Literal
-
-import pyautogui as gui
-
-LOCATE_AND_CLICK_DELAY = 0.2
-
-
-def locate_and_click(
-    filename: str, clicks: int = 1, button: Literal["left", "right"] = "left"
-):
-    time.sleep(LOCATE_AND_CLICK_DELAY)
-    locate = gui.locateOnScreen(filename, grayscale=True)
-    assert locate is not None, f"Could not locate {filename} on screen."
-    locate_center = (locate.left + locate.width // 2), (locate.top + locate.height // 2)
-    gui.moveTo(*locate_center, 0.6, gui.easeInOutQuad)  # type: ignore
-    gui.click(clicks=clicks, button=button)
-    time.sleep(LOCATE_AND_CLICK_DELAY)
-
-
-def is_valid_variable_name(name):
-    return name.isidentifier() and not iskeyword(name)
+from __future__ import annotations
+
+import logging
+import os
+import time
+from keyword import iskeyword
+from typing import Callable, overload
+
+import numpy as np
+import pyautogui as gui
+from PIL import Image
+
+from ._types import MouseButton, TowardsDirection
+from .constants import LOCATE_AND_CLICK_DELAY, POINTER_SPEED
+from .region import Region, RegionSpec
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def locate_and_click(
+    reference: Image.Image | str,
+    clicks: int = 1,
+    button: MouseButton = "left",
+    region: RegionSpec | None = None,
+    confidence: float = 0.999,
+    grayscale: bool = True,
+    limit: int = 1,
+    offset: tuple[int, int] = (0, 0),
+    towards: TowardsDirection | None = None,
+):
+    time.sleep(LOCATE_AND_CLICK_DELAY)
+    found_region = locate_on_screen(
+        reference,
+        region=region,
+        confidence=confidence,
+        grayscale=grayscale,
+        limit=limit,
+    )
+    assert found_region is not None, f"Could not locate {reference} on screen."
+    move_and_click(
+        found_region, clicks=clicks, button=button, offset=offset, towards=towards
+    )
+    time.sleep(LOCATE_AND_CLICK_DELAY)
+
+
+def move_and_click(
+    target_region: RegionSpec,
+    clicks: int = 1,
+    button: MouseButton = "left",
+    offset: tuple[int, int] = (0, 0),
+    towards: TowardsDirection | None = None,
+):
+    """Move to the center or edge of the region and click.
+
+    The offset is always added to the calculated target point.
+    For example, for 'bottom', offset=(0, 5) means 5 pixels below the bottom edge.
+    """
+    _target_region = Region.from_spec(target_region)
+    base_points = {
+        "top": (_target_region.center[0], _target_region.top),
+        "left": (_target_region.left, _target_region.center[1]),
+        "bottom": (
+            _target_region.center[0],
+            _target_region.top + _target_region.height - 1,
+        ),
+        "right": (
+            _target_region.left + _target_region.width - 1,
+            _target_region.center[1],
+        ),
+        None: _target_region.center,
+    }
+    if towards not in base_points:
+        raise ValueError(f"Invalid direction: {towards}")
+    base = base_points[towards]
+    target = (base[0] + offset[0], base[1] + offset[1])
+
+    current = gui.position()
+    duration = np.linalg.norm(np.array(target) - np.array(current)) / POINTER_SPEED
+    gui.moveTo(*target, float(duration), gui.easeInOutQuad)  # type: ignore
+    gui.click(clicks=clicks, button=button)
+
+
+def is_valid_variable_name(name):
+    return name.isidentifier() and not iskeyword(name)
+
+
+@overload
+def locate_on_screen(
+    reference: Image.Image | str,
+    region: RegionSpec | None = None,
+    confidence: float = 0.999,
+    grayscale: bool = True,
+    limit: int = 1,
+    locator: Callable[[Image.Image, Image.Image], list[Region]] | None = None,
+) -> Region | None: ...
+@overload
+def locate_on_screen(
+    reference: Image.Image | str,
+    region: RegionSpec | None = None,
+    confidence: float = 0.999,
+    grayscale: bool = True,
+    limit: int = 1,
+    locator: Callable[[Image.Image, Image.Image], list[Region]] | None = None,
+) -> list[Region] | None: ...
+def locate_on_screen(
+    reference: Image.Image | str,
+    region: RegionSpec | None = None,
+    confidence: float = 0.999,
+    grayscale: bool = True,
+    limit: int = 1,
+    locator: Callable[[Image.Image, Image.Image], list[Region]] | None = None,
+):
+    """Locate a region on the screen."""
+    if isinstance(reference, str):
+        if not os.path.exists(reference):
+            raise FileNotFoundError(f"Image file {reference} does not exist.")
+        reference = Image.open(reference)
+    if locator is None:
+        try:
+            location = gui.locateOnScreen(
+                reference,
+                region=Region.from_spec(region).to_box() if region else None,
+                grayscale=grayscale,
+                confidence=confidence,
+                limit=limit,
+            )
+            if location:
+                return Region.from_box(location)
+        except gui.ImageNotFoundException:
+            return None
+        except FileNotFoundError:
+            return None
+    else:
+        screenshot = gui.screenshot(
+            region=Region.from_spec(region).to_box() if region else None
+        )
+        logger.info(
+            f"Searching in region: {Region.from_spec(region).to_box() if region else None}.\nGiven region: {region}"
+        )
+        detections = locator(reference, screenshot)
+        logger.info("total detections: %d", len(detections))
+        if len(detections) == 0:
+            return None
+        elif limit > 1:
+            return detections[:limit]
+        else:
+            return detections[0]

{pyautoscene-0.2.0.dist-info → pyautoscene-0.2.2.dist-info}/METADATA

@@ -1,14 +1,18 @@
 Metadata-Version: 2.4
 Name: pyautoscene
-Version: 0.2.0
+Version: 0.2.2
 Summary: Advance GUI automation
 Author-email: pritam-dey3 <pritam.pritamdey.984@gmail.com>
 License-File: LICENSE
 Requires-Python: >=3.13
 Requires-Dist: networkx>=3.5
+Requires-Dist: opencv-python-headless>=4.12.0.88
 Requires-Dist: pillow>=11.3.0
 Requires-Dist: pyautogui>=0.9.54
 Requires-Dist: python-statemachine[diagrams]
+Provides-Extra: ocr
+Requires-Dist: onnxruntime>=1.22.0; extra == 'ocr'
+Requires-Dist: rapidocr>=3.2.0; extra == 'ocr'
 Description-Content-Type: text/markdown
 
 # PyAutoScene

pyautoscene-0.2.2.dist-info/RECORD

@@ -0,0 +1,15 @@
+pyautoscene/__init__.py,sha256=9w2x28VgAHsTZHeLIynHs0qcFfmdUM-FjURLwUahNO0,230
+pyautoscene/_types.py,sha256=t6hRAUe9OLUVCpzEVRjYHgb5l5iu4gMGinB_ecNOkR4,180
+pyautoscene/constants.py,sha256=ZkBdvW9gBw002hLqelgiyS-ZsTbu7juyEWEg4Fx16NE,219
+pyautoscene/ocr.py,sha256=YbPNYXGIvM_5RSB4_Va6D1t0eWIo47R-0utrBCZLqAk,2322
+pyautoscene/ocr_config.yaml,sha256=EGaKX1a-LyWk0gtI2wUo04LraFqGJ3aZnQ5NCPbgDYI,2231
+pyautoscene/references.py,sha256=8-VgwAHUt9Te0np2O5TvnIzW5LhNY1Dzm222VdlxgK8,3338
+pyautoscene/region.py,sha256=X2QQy6gIqbbFPWVTNiqbYaQ12nXv5MU-ptP20kA4u48,2123
+pyautoscene/scene.py,sha256=VWtx5l9DMFu3TC-EAxhabREQDFCvUPs7-4Jxq35JDrI,1993
+pyautoscene/session.py,sha256=lxv71Q4UW5xr3-UTYVSOV8gqbpDI5HXMopO5kUSZhM0,5678
+pyautoscene/utils.py,sha256=ZaxlGlt3xjayCKUfWGr_C73QwqEsacOsnp0SYC6ao7w,4734
+pyautoscene-0.2.2.dist-info/METADATA,sha256=N4woAGtRHFc2_DNHG3b_g7jXA3HYT6DB1cu369vVtNs,6068
+pyautoscene-0.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pyautoscene-0.2.2.dist-info/entry_points.txt,sha256=6aKjylfDivCRMrJasIIi7ICU4fZR-8HjcOHhRmHpYpQ,49
+pyautoscene-0.2.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pyautoscene-0.2.2.dist-info/RECORD,,