ok-script 0.0.7__tar.gz

ok-script-0.0.7/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: ok-script
+Version: 0.0.7
+Summary: Automation with Computer Vision for Python
+Home-page: https://github.com/ok-oldking/ok-script
+Author: ok-oldking
+Author-email: firedcto@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: Microsoft :: Windows
+Requires-Python: >=3.4
+Description-Content-Type: text/markdown
+Requires-Dist: adbutils==2.2.1
+Requires-Dist: numpy~=1.26.4
+Requires-Dist: PyDirectInput~=1.0.4
+Requires-Dist: pywin32==306
+Requires-Dist: typing-extensions~=4.10.0
+Requires-Dist: PySide6~=6.6.2
+
+# autoui
+
+This project aims to provide a universal framework for developing automation tool for PC/Android games, running in
+windows,
+emulator, or physical Android Phone. Currently still working in progress, there is a working
+
+## Showcase
+
+![docs/img_showcase.png](docs/img_showcase.png)
+
+## Example Usage
+
+Basically you only need to write the Tasks and Scenes, the autoui framework will do all the heavy lifting.
+[genshin/main.py](genshin/main.py)
+
+```python
+# Defining game scenes to handle different in-game situations through automated tasks
+task_executor.scenes.extend([
+    WorldScene(interaction, feature_set),
+    StartScene(interaction, feature_set),
+    MonthlyCardScene(interaction, feature_set),
+    DialogCloseButtonScene(interaction, feature_set),
+    DialogChoicesScene(interaction, feature_set),
+    DialogPlayingScene(interaction, feature_set),
+    BlackDialogScene(interaction, feature_set),
+])
+
+# Adding automated tasks for gameplay, such as dialog navigation and item collection
+task_executor.tasks.extend([
+    AutoPlayDialogTask(interaction, feature_set),  # speeding up the dialogs
+    AutoChooseDialogTask(interaction, feature_set),  # choose dialog options
+    AutoPickTask(interaction, feature_set),  # pickup items in world scene
+    AutoLoginTask(interaction, feature_set),  # auto login and claim rewards
+])
+```
+
+## Scenes
+
+## Tasks
+
+## Project Structure
+
+- `autoui`: Framework code.
+- `genshin`: Example Genshin Impact automation project

ok-script-0.0.7/README.md

@@ -0,0 +1,44 @@
+# autoui
+
+This project aims to provide a universal framework for developing automation tool for PC/Android games, running in
+windows,
+emulator, or physical Android Phone. Currently still working in progress, there is a working
+
+## Showcase
+
+![docs/img_showcase.png](docs/img_showcase.png)
+
+## Example Usage
+
+Basically you only need to write the Tasks and Scenes, the autoui framework will do all the heavy lifting.
+[genshin/main.py](genshin/main.py)
+
+```python
+# Defining game scenes to handle different in-game situations through automated tasks
+task_executor.scenes.extend([
+    WorldScene(interaction, feature_set),
+    StartScene(interaction, feature_set),
+    MonthlyCardScene(interaction, feature_set),
+    DialogCloseButtonScene(interaction, feature_set),
+    DialogChoicesScene(interaction, feature_set),
+    DialogPlayingScene(interaction, feature_set),
+    BlackDialogScene(interaction, feature_set),
+])
+
+# Adding automated tasks for gameplay, such as dialog navigation and item collection
+task_executor.tasks.extend([
+    AutoPlayDialogTask(interaction, feature_set),  # speeding up the dialogs
+    AutoChooseDialogTask(interaction, feature_set),  # choose dialog options
+    AutoPickTask(interaction, feature_set),  # pickup items in world scene
+    AutoLoginTask(interaction, feature_set),  # auto login and claim rewards
+])
+```
+
+## Scenes
+
+## Tasks
+
+## Project Structure
+
+- `autoui`: Framework code.
+- `genshin`: Example Genshin Impact automation project

ok-script-0.0.7/autohelper/AutoHelper.py

@@ -0,0 +1,116 @@
+import sys
+import threading
+import time
+from typing import Dict, Any
+
+from PySide6.QtWidgets import QApplication
+
+import autohelper
+from autohelper.feature.FeatureSet import FeatureSet
+from autohelper.gui.App import App
+from autohelper.gui.Communicate import communicate
+from autohelper.gui.overlay.OverlayWindow import OverlayWindow
+from autohelper.gui.util.InitWorker import InitWorker
+from autohelper.logging.Logger import get_logger, config_logger
+from autohelper.task.TaskExecutor import TaskExecutor
+
+logger = get_logger(__name__)
+
+
+class AutoHelper:
+    executor: TaskExecutor
+    adb = None
+    adb_device = None
+    feature_set = None
+    hwnd = None
+    device_manager = None
+    ocr = None
+    overlay_window = None
+    app = None
+
+    def __init__(self, config: Dict[str, Any]):
+        print(f"AutoHelper init, config: {config}")
+        self.debug = config.get("debug", False)
+        self.exit_event = threading.Event()
+        self.config = config
+        self.init_device_manager()
+        if config.get("use_gui"):
+            self.app = App(self.config.get('gui_icon'), self.debug, self.config.get('gui_title'),
+                           self.config['tasks'], self.exit_event)
+            autohelper.gui.app = self.app
+            self.app.show_loading()
+            self.worker = InitWorker(self.do_init)
+            self.worker.start()
+            self.app.exec()
+        else:
+            self.device_manager.set_preferred_device()
+            self.device_manager.start()
+            self.do_init()
+            self.task_executor.start()
+            if config.get("debug"):
+                self.app = QApplication(sys.argv)
+                self.overlay_window = OverlayWindow(autohelper.gui.device_manager.hwnd)
+                self.app.exec()
+            else:
+                try:
+                    # Starting the task in a separate thread (optional)
+                    # This allows the main thread to remain responsive to keyboard interrupts
+                    task_thread = threading.Thread(target=self.wait_task())
+                    task_thread.start()
+
+                    # Wait for the task thread to end (which it won't, in this case, without an interrupt)
+                    task_thread.join()
+                except KeyboardInterrupt:
+                    self.exit_event.set()
+                    logger.info("Keyboard interrupt received, exiting script.")
+                finally:
+                    # Clean-up code goes here (if any)
+                    # This block ensures that the script terminates gracefully,
+                    # releasing resources or performing necessary clean-up operations.
+                    logger.info("Script has terminated.")
+
+    def init_message(self, message: str, done=False):
+        communicate.init.emit(done, message)
+        if self.exit_event.is_set():
+            self.worker.quit()
+
+    def do_init(self):
+        logger.info(f"initializing {self.__class__.__name__}, config: {self.config}")
+        if self.config.get('ocr'):
+            self.init_message("RapidOCR init Start")
+            from rapidocr_onnxruntime import RapidOCR
+            self.ocr = RapidOCR()
+            self.init_message("RapidOCR init Complete")
+
+        config_logger(self.config)
+
+        if self.config.get('coco_feature_folder') is not None:
+            self.init_message("FeatureSet init Start")
+            coco_feature_folder = self.config.get('coco_feature_folder')
+            self.feature_set = FeatureSet(coco_feature_folder,
+                                          default_horizontal_variance=self.config.get('default_horizontal_variance', 0),
+                                          default_vertical_variance=self.config.get('default_vertical_variance', 0),
+                                          default_threshold=self.config.get('default_threshold', 0))
+            self.init_message("FeatureSet init Complete")
+
+        self.init_message("TaskExecutor init Start")
+        self.task_executor = TaskExecutor(self.device_manager, exit_event=self.exit_event,
+                                          tasks=self.config['tasks'], scenes=self.config['scenes'],
+                                          feature_set=self.feature_set,
+                                          ocr=self.ocr, config_folder=self.config.get("config_folder") or "config")
+        self.init_message("TaskExecutor init Done")
+
+        if self.app:
+            autohelper.gui.executor = self.task_executor
+
+    def wait_task(self):
+        while not self.exit_event.is_set():
+            time.sleep(1)
+
+    def init_device_manager(self):
+        if self.device_manager is None:
+            from autohelper.capture.adb.DeviceManager import DeviceManager
+            self.device_manager = DeviceManager(self.config.get("config_folder") or "config",
+                                                self.config.get('capture_window_title'), self.config.get("debug"),
+                                                self.exit_event)
+            autohelper.gui.device_manager = self.device_manager

ok-script-0.0.7/autohelper/capture/BaseCaptureMethod.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+
+class BaseCaptureMethod:
+    name = "None"
+    description = ""
+    last_captured_frame: np.ndarray
+    top_cut = 0
+    bottom_cut = 0
+    left_cut = 0
+    right_cut = 0
+    width = 0
+    height = 0
+
+    def __init__(self):
+        # Some capture methods don't need an initialization process
+        pass
+
+    def close(self):
+        # Some capture methods don't need an initialization process
+        pass
+
+    def get_frame(self) -> np.ndarray | None:
+        pass
+
+    def draw_rectangle(self):
+        pass
+
+    def clickable(self):
+        pass

ok-script-0.0.7/autohelper/capture/HwndWindow.py

@@ -0,0 +1,138 @@
+# original https://github.com/dantmnf & https://github.com/hakaboom/winAuto
+import re
+import threading
+
+from typing_extensions import override
+from win32 import win32gui
+
+from autohelper.capture.windows.window import is_foreground_window, get_window_bounds
+from autohelper.gui.Communicate import communicate
+from autohelper.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class HwndWindow:
+    visible = True
+    x = 0
+    y = 0
+    width = 0
+    height = 0
+    title_height = 0
+    border = 0
+    scaling = 1
+    top_cut = 0
+    right_cut = 0
+    bottom_cut = 0
+    left_cut = 0
+    window_change_listeners = []
+    frame_aspect_ratio = 0
+    hwnd = None
+    frame_width = 0
+    frame_height = 0
+    exists = False
+
+    def __init__(self, title="", exit_event=threading.Event(), frame_width=0, frame_height=0):
+        super().__init__()
+        self.app_exit_event = exit_event
+        self.stop_event = threading.Event()
+        self.update_title_re(title)
+        self.visible = False
+        self.update_frame_size(frame_width, frame_height)
+        self.do_update_window_size()
+        self.thread = threading.Thread(target=self.update_window_size)
+        self.thread.start()
+
+    @override
+    def close(self):
+        self.app_exit_event.set()
+
+    def stop(self):
+        self.stop_event.set()
+
+    def update_title_re(self, title):
+        self.title = re.compile(title)
+
+    def update_frame_size(self, width, height):
+        if width != self.frame_width or height != self.frame_height:
+            self.frame_width = width
+            self.frame_height = height
+            if width > 0 and height > 0:
+                self.frame_aspect_ratio = width / height
+                logger.debug(f"HwndWindow: frame ratio:{self.frame_aspect_ratio} width: {width}, height: {height}")
+
+    def update_window_size(self):
+        while not self.app_exit_event.is_set() and not self.stop_event.is_set():
+            self.do_update_window_size()
+            self.app_exit_event.wait(0.1)
+
+    def get_abs_cords(self, x, y):
+        return int(self.x * self.scaling + (self.border * self.scaling + x)), int(
+            self.y * self.scaling + (y + self.title_height * self.scaling))
+
+    def get_top_left_frame_offset(self):
+        return int(self.border * self.scaling), int(
+            self.title_height * self.scaling)
+
+    def do_update_window_size(self):
+        visible, x, y, border, title_height, width, height, scaling = self.visible, self.x, self.y, self.border, self.title_height, self.width, self.height, self.scaling
+        if self.hwnd is None:
+            self.hwnd = find_hwnds_by_title(self.title)
+        if self.hwnd is not None:
+            self.exists = win32gui.IsWindow(self.hwnd)
+            if self.exists:
+                visible = is_foreground_window(self.hwnd)
+                x, y, border, title_height, width, height, scaling = get_window_bounds(
+                    self.hwnd)
+                if self.frame_aspect_ratio != 0:
+                    window_ratio = width / height
+                    if window_ratio < self.frame_aspect_ratio:
+                        cropped_window_height = int(width / self.frame_aspect_ratio)
+                        title_height += height - cropped_window_height
+                        height = cropped_window_height
+                height = height
+                width = width
+                title_height = title_height
+            else:
+                self.hwnd = None
+            changed = False
+            if visible != self.visible or self.scaling != scaling:
+                self.visible = visible
+                self.scaling = scaling
+                changed = True
+            if (
+                    x != self.x or y != self.y or border != self.border or title_height != self.title_height or width != self.width or height != self.height or scaling != self.scaling) and (
+                    (x >= 0 and y >= 0) or self.visible):
+                self.x, self.y, self.border, self.title_height, self.width, self.height = x, y, border, title_height, width, height
+                changed = True
+            if changed:
+                logger.debug(
+                    f"do_update_window_size changed: {self.visible} {self.x} {self.y} {self.border} {self.width} {self.height} {self.scaling}")
+                communicate.window.emit(self.visible, self.x, self.y, self.border, self.title_height, self.width,
+                                        self.height, self.scaling)
+
+    def frame_ratio(self, size):
+        if self.frame_width > 0 and self.width > 0:
+            return int(size / self.frame_width * self.width)
+        else:
+            return size
+
+    def title_text(self):
+        if self.hwnd:
+            return win32gui.GetWindowText(self.hwnd)
+        return ""
+
+
+def find_hwnds_by_title(title):
+    hwnds = []
+
+    def enum_windows_proc(hwnd, lParam):
+        if win32gui.IsWindowVisible(hwnd) and win32gui.GetWindowText(hwnd):
+            if re.search(title, win32gui.GetWindowText(hwnd)):
+                hwnds.append(hwnd)
+
+    win32gui.EnumWindows(enum_windows_proc, None)
+    if len(hwnds) > 0:
+        if len(hwnds) > 1:
+            logger.warning(f"Found multiple hwnds {len(hwnds)}")
+        return hwnds[0]

ok-script-0.0.7/autohelper/capture/adb/ADBCaptureMethod.py

@@ -0,0 +1,29 @@
+# original https://github.com/Toufool/AutoSplit/blob/master/src/capture_method/WindowsGraphicsCaptureMethod.py
+import numpy as np
+from typing_extensions import override
+
+from autohelper.capture.BaseCaptureMethod import BaseCaptureMethod
+from autohelper.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ADBCaptureMethod(BaseCaptureMethod):
+    name = "ADB command line Capture"
+    description = "use the adb screencap command, slow but works when in background/minimized, takes 300ms per frame"
+
+    def __init__(self, device_manager):
+        super().__init__()
+        self.device_manager = device_manager
+
+    @property
+    def width(self):
+        return self.device_manager.width
+
+    @property
+    def height(self):
+        return self.device_manager.height
+
+    @override
+    def get_frame(self) -> np.ndarray | None:
+        return self.device_manager.screencap()

ok-script-0.0.7/autohelper/capture/adb/DeviceManager.py

@@ -0,0 +1,209 @@
+import threading
+
+import adbutils
+import cv2
+import numpy as np
+
+from autohelper.capture.HwndWindow import HwndWindow
+from autohelper.capture.adb.ADBCaptureMethod import ADBCaptureMethod
+from autohelper.capture.adb.vbox import installed_emulator
+from autohelper.capture.windows.WindowsGraphicsCaptureMethod import WindowsGraphicsCaptureMethod
+from autohelper.config.Config import Config
+from autohelper.gui.Communicate import communicate
+from autohelper.interaction.ADBInteraction import ADBBaseInteraction
+from autohelper.interaction.Win32Interaction import Win32Interaction
+from autohelper.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class DeviceManager:
+
+    def __init__(self, config_folder, hwnd_title=None, debug=False, exit_event=None):
+        self._device = None
+        self.adb = None
+        self.hwnd_title = hwnd_title
+        self.debug = debug
+        self.interaction = None
+        self.exit_event = exit_event
+        if hwnd_title is not None:
+            self.hwnd = HwndWindow(hwnd_title, exit_event)
+        self.config = Config({"devices": {}}, config_folder, "DeviceManager")
+        self._size = (0, 0)
+        self.thread = None
+        self.capture_method = None
+        self.refresh()
+
+    def refresh(self):
+        if self.thread is None or not self.thread.is_alive():
+            self.thread = threading.Thread(target=self.do_refresh, name="refresh adb")
+            self.thread.start()
+
+    @property
+    def device_dict(self):
+        return self.config.get("devices")
+
+    def connect_all(self):
+        for device in self.device_dict.values():
+            self.adb.connect(device['address'])
+        logger.debug(f'connect_all {self.adb.device_list()}')
+
+    def get_devices(self):
+        return self.device_dict.values()
+
+    def do_refresh(self):
+        if self.adb is None:
+            self.adb = adbutils.AdbClient(host="127.0.0.1")
+            logger.debug(f'connect adb')
+        self.connect_all()
+        installed_emulators = installed_emulator()
+        for device in self.adb.list():
+            logger.debug(f'adb.list() {device}')
+
+        for emulator in installed_emulators:
+            logger.debug(f"installed_emulator: {emulator}")
+            self.adb.connect(emulator.adb_address)
+
+        device_list = self.adb.device_list()
+
+        device_list = sorted(device_list, key=lambda x: x.serial)
+
+        for device in self.device_dict.values():
+            device['connected'] = False
+
+        for device in device_list:
+            imei = device.shell("settings get secure android_id") or device.shell(
+                "service call iphonesubinfo 4") or device.prop.model
+            frame = do_screencap(device)
+            width, height = 0, 0
+            if frame is not None:
+                height, width, _ = frame.shape
+            adb_device = {"address": device.serial, "imei": imei, "method": "adb",
+                          "model": device.prop.model, "nick": device.prop.model, "connected": True, "preferred": False,
+                          "resolution": f"{width}x{height}"}
+            found = False
+            for emulator in installed_emulators:
+                if emulator.adb_address == adb_device['address']:
+                    adb_device['nick'] = emulator.description
+                    found = True
+            if self.device_dict.get(imei):
+                if found or not self.device_dict[imei]['connected']:
+                    self.device_dict[imei] = adb_device
+            else:
+                self.device_dict[imei] = adb_device
+        if self.hwnd_title:
+            nick = self.hwnd_title
+            width = 0
+            height = 0
+            if self.hwnd:
+                nick = self.hwnd.title_text()
+                width = self.hwnd.width
+                height = self.hwnd.height
+            self.device_dict[self.hwnd_title] = {"address": "", "imei": self.hwnd_title, "method": "windows",
+                                                 "model": "", "nick": nick or self.hwnd_title,
+                                                 "connected": self.hwnd is not None and self.hwnd.exists,
+                                                 "preferred": False,
+                                                 "resolution": f"{width}x{height}"}
+        communicate.adb_devices.emit()
+        self.config.save_file()
+        logger.debug(f'refresh {self.device_dict}')
+
+    def set_preferred_device(self, imei=None):
+        logger.debug(f"set_preferred_device {imei}")
+        preferred = None
+        if imei is None:
+            for device in self.device_dict.values():
+                if device.get('preferred'):
+                    preferred = device
+                    imei = preferred['imei']
+        else:
+            preferred = self.device_dict.get(imei)
+        for key in self.device_dict.keys():
+            if imei == key:
+                self.device_dict[imei]["preferred"] = False
+        if preferred is None and len(self.device_dict) > 0:
+            preferred = next(iter(self.device_dict.values()))
+        if preferred is not None:
+            preferred['preferred'] = True
+            if preferred['method'] == 'windows':
+                if not isinstance(self.capture_method, WindowsGraphicsCaptureMethod):
+                    if self.capture_method is not None:
+                        self.capture_method.close()
+                    self.capture_method = WindowsGraphicsCaptureMethod(self.hwnd)
+                    self.interaction = Win32Interaction(self.capture_method)
+                self.capture_method.hwnd_window = self.hwnd
+            else:
+                for adb_device in self.adb.device_list():
+                    if adb_device.serial == preferred.get('address'):
+                        self._device = adb_device
+                if not isinstance(self.capture_method, ADBCaptureMethod):
+                    if self.capture_method is not None:
+                        self.capture_method.close()
+                    self.capture_method = ADBCaptureMethod(self)
+                    self.interaction = ADBBaseInteraction(self, self.capture_method)
+        self.config.save_file()
+        logger.debug(f'preferred device: {preferred}')
+
+    def start(self):
+        if isinstance(self.capture_method, ADBCaptureMethod):
+            if self.debug:
+                if self.hwnd is not None:
+                    self.hwnd.update_title_re("autohelper_debug")
+                    self.hwnd.update_frame_size(self.capture_method.width, self.capture_method.height)
+                else:
+                    self.hwnd = HwndWindow("autohelper_debug", self.exit_event, self.capture_method.width,
+                                           self.capture_method.height)
+            else:
+                self.hwnd.stop()
+                self.hwnd = None
+
+    @property
+    def device(self):
+        if self._device is None:
+            self.set_preferred_device()
+        return self._device
+
+    @property
+    def width(self):
+        if self._size[0] == 0:
+            self.screencap()
+        return self._size[0]
+
+    @property
+    def height(self):
+        if self._size[1] == 0:
+            self.screencap()
+        return self._size[1]
+
+    def update_device_list(self):
+        pass
+
+    def shell(self, *args, **kwargs):
+        device = self.device
+        if device is not None:
+            try:
+                return device.shell(*args, **kwargs)
+            except Exception as e:
+                logger.error(f"shell_wrapper error occurred: {e}")
+
+    def screencap(self):
+        if self.exit_event.is_set():
+            return None
+        self._size = (0, 0)
+        frame = do_screencap(self.device)
+        if frame is not None:
+            height, width, _ = frame.shape
+            self._size = (width, height)
+        return frame
+
+
+def do_screencap(device) -> np.ndarray | None:
+    if device is not None:
+        png_bytes = device.shell("screencap -p", encoding=None)
+        if png_bytes is not None and len(png_bytes) > 0:
+            image_data = np.frombuffer(png_bytes, dtype=np.uint8)
+            image = cv2.imdecode(image_data, cv2.IMREAD_COLOR)
+            if image is not None:
+                return image
+            else:
+                logger.error(f"Screencap image decode error, probably disconnected")