ok-script 0.0.9__tar.gz → 0.0.10__tar.gz

{ok-script-0.0.9 → ok-script-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ok-script
-Version: 0.0.9
+Version: 0.0.10
 Summary: Automation with Computer Vision for Python
 Home-page: https://github.com/ok-oldking/ok-script
 Author: ok-oldking
@@ -20,13 +20,16 @@ Requires-Dist: PySide6~=6.6.2
 
 # ok-script
 
-This project aims to provide a universal framework for developing automation tool for PC/Android games, running in
-windows,
-emulator, or physical Android Phone. Currently still working in progress, there is a working
+类似于AirScript, 使用Python
 
-## Showcase
+## 优势
 
-![docs/img_showcase.png](docs/img_showcase.png)
+1. 开源免费, 依赖库均为开源方案, 支持pip install任何第三方库
+2. 代码和打包流程完全由开发者控制, 不会上传服务器, 没有安全性隐患, 不容易被侦测和识别
+3. 一套代码即可支持Windows安卓模拟器/ADB连接的虚拟机, Windows客户端游戏, 安卓系统直接运行(开发中)
+4. 支持GUI以及Console运行, 也可以完全自己开发GUI
+5. 支持Python3.7以上版本
+6. 学习成本低, 自定义函数少, 也可以直接使用opencv, RapidOCR等api
 
 ## Example Usage
 

{ok-script-0.0.9 → ok-script-0.0.10}/README.md

@@ -1,12 +1,15 @@
 # ok-script
 
-This project aims to provide a universal framework for developing automation tool for PC/Android games, running in
-windows,
-emulator, or physical Android Phone. Currently still working in progress, there is a working
+类似于AirScript, 使用Python
 
-## Showcase
+## 优势
 
-![docs/img_showcase.png](docs/img_showcase.png)
+1. 开源免费, 依赖库均为开源方案, 支持pip install任何第三方库
+2. 代码和打包流程完全由开发者控制, 不会上传服务器, 没有安全性隐患, 不容易被侦测和识别
+3. 一套代码即可支持Windows安卓模拟器/ADB连接的虚拟机, Windows客户端游戏, 安卓系统直接运行(开发中)
+4. 支持GUI以及Console运行, 也可以完全自己开发GUI
+5. 支持Python3.7以上版本
+6. 学习成本低, 自定义函数少, 也可以直接使用opencv, RapidOCR等api
 
 ## Example Usage
 

{ok-script-0.0.9 → ok-script-0.0.10}/ok/OK.py

@@ -9,6 +9,7 @@ import ok
 from ok.feature.FeatureSet import FeatureSet
 from ok.gui.App import App
 from ok.gui.Communicate import communicate
+from ok.gui.debug.Screenshot import Screenshot
 from ok.gui.overlay.OverlayWindow import OverlayWindow
 from ok.gui.util.InitWorker import InitWorker
 from ok.logging.Logger import get_logger, config_logger
@@ -27,16 +28,18 @@ class OK:
     ocr = None
     overlay_window = None
     app = None
+    screenshot = None
 
     def __init__(self, config: Dict[str, Any]):
         print(f"AutoHelper init, config: {config}")
+        ok.gui.ok = self
         self.debug = config.get("debug", False)
         self.exit_event = threading.Event()
         self.config = config
         self.init_device_manager()
+        self.screenshot = Screenshot()
         if config.get("use_gui"):
-            self.app = App(self.config.get('gui_icon'), self.debug, self.config.get('gui_title'),
-                           self.config['tasks'], self.config.get('about'), self.exit_event)
+            self.app = App(config, self.exit_event)
             ok.gui.app = self.app
         else:
             self.device_manager.set_preferred_device()

{ok-script-0.0.9 → ok-script-0.0.10}/ok/capture/HwndWindow.py

@@ -54,6 +54,7 @@ class HwndWindow:
         self.title = re.compile(title)
 
     def update_frame_size(self, width, height):
+        logger.debug(f"update_frame_size:{self.frame_width}x{self.frame_height} to {width}x{height}")
         if width != self.frame_width or height != self.frame_height:
             self.frame_width = width
             self.frame_height = height
@@ -127,8 +128,11 @@ def find_hwnds_by_title(title):
     hwnds = []
 
     def enum_windows_proc(hwnd, lParam):
-        if win32gui.IsWindowVisible(hwnd) and win32gui.GetWindowText(hwnd):
-            if re.search(title, win32gui.GetWindowText(hwnd)):
+        text = win32gui.GetWindowText(hwnd)
+        if text and win32gui.IsWindowVisible(hwnd):
+            if isinstance(title, str) and title == text:
+                hwnds.append(hwnd)
+            elif re.search(title, text):
                 hwnds.append(hwnd)
 
     win32gui.EnumWindows(enum_windows_proc, None)

ok-script-0.0.10/ok/capture/adb/ADBCaptureMethod.py

@@ -0,0 +1,57 @@
+# original https://github.com/Toufool/AutoSplit/blob/master/src/capture_method/WindowsGraphicsCaptureMethod.py
+import cv2
+import numpy as np
+from typing_extensions import override
+
+from ok.capture.BaseCaptureMethod import BaseCaptureMethod
+from ok.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ADBCaptureMethod(BaseCaptureMethod):
+    name = "ADB command line Capture"
+    description = "use the adb screencap command, slow but works when in background/minimized, takes 300ms per frame"
+
+    def __init__(self, device, exit_event, width=0, height=0):
+        super().__init__()
+        self.exit_event = exit_event
+        self._size = (width, height)
+        self.device = device
+
+    @property
+    def width(self):
+        if self._size[0] == 0:
+            self.screencap()
+        return self._size[0]
+
+    @property
+    def height(self):
+        if self._size[1] == 0:
+            self.screencap()
+        return self._size[1]
+
+    @override
+    def get_frame(self) -> np.ndarray | None:
+        return self.screencap()
+
+    def screencap(self):
+        if self.exit_event.is_set():
+            return None
+        frame = do_screencap(self.device)
+        if frame is not None:
+            height, width, _ = frame.shape
+            self._size = (width, height)
+        return frame
+
+
+def do_screencap(device) -> np.ndarray | None:
+    if device is not None:
+        png_bytes = device.shell("screencap -p", encoding=None)
+        if png_bytes is not None and len(png_bytes) > 0:
+            image_data = np.frombuffer(png_bytes, dtype=np.uint8)
+            image = cv2.imdecode(image_data, cv2.IMREAD_COLOR)
+            if image is not None:
+                return image
+            else:
+                logger.error(f"Screencap image decode error, probably disconnected")

{ok-script-0.0.9 → ok-script-0.0.10}/ok/capture/adb/DeviceManager.py

@@ -1,11 +1,9 @@
 import threading
 
 import adbutils
-import cv2
-import numpy as np
 
 from ok.capture.HwndWindow import HwndWindow
-from ok.capture.adb.ADBCaptureMethod import ADBCaptureMethod
+from ok.capture.adb.ADBCaptureMethod import ADBCaptureMethod, do_screencap
 from ok.capture.adb.vbox import installed_emulator
 from ok.capture.windows.WindowsGraphicsCaptureMethod import WindowsGraphicsCaptureMethod
 from ok.config.Config import Config
@@ -29,7 +27,6 @@ class DeviceManager:
         if hwnd_title is not None:
             self.hwnd = HwndWindow(hwnd_title, exit_event)
         self.config = Config({"devices": {}}, config_folder, "DeviceManager")
-        self._size = (0, 0)
         self.thread = None
         self.capture_method = None
         self.refresh()
@@ -68,8 +65,7 @@ class DeviceManager:
 
         device_list = sorted(device_list, key=lambda x: x.serial)
 
-        for device in self.device_dict.values():
-            device['connected'] = False
+        adb_connected = []
 
         for device in device_list:
             imei = device.shell("settings get secure android_id") or device.shell(
@@ -78,7 +74,8 @@ class DeviceManager:
             width, height = 0, 0
             if frame is not None:
                 height, width, _ = frame.shape
-            adb_device = {"address": device.serial, "imei": imei, "method": "adb",
+            adb_device = {"address": device.serial, "imei": imei, "device": "adb", "capture": "adb", "width": width,
+                          "height": height,
                           "model": device.prop.model, "nick": device.prop.model, "connected": True, "preferred": False,
                           "resolution": f"{width}x{height}"}
             found = False
@@ -88,19 +85,31 @@ class DeviceManager:
                     found = True
             if self.device_dict.get(imei):
                 if found or not self.device_dict[imei]['connected']:
+                    old_capture = self.device_dict[imei].get("capture")
+                    old_hwnd = self.device_dict[imei].get("hwnd")
                     self.device_dict[imei] = adb_device
+                    if old_capture is not None:
+                        self.device_dict[imei]['capture'] = old_capture
+                    if old_hwnd is not None:
+                        self.device_dict[imei]['hwnd'] = old_hwnd
             else:
                 self.device_dict[imei] = adb_device
+            adb_connected.append(imei)
+        for imei in self.device_dict:
+            if imei not in adb_connected:
+                self.device_dict[imei]['connected'] = False
         if self.hwnd_title:
-            nick = self.hwnd_title
+            nick = ""
             width = 0
             height = 0
             if self.hwnd:
                 nick = self.hwnd.title_text()
                 width = self.hwnd.width
                 height = self.hwnd.height
-            self.device_dict[self.hwnd_title] = {"address": "", "imei": self.hwnd_title, "method": "windows",
-                                                 "model": "", "nick": nick or self.hwnd_title,
+            self.device_dict[self.hwnd_title] = {"address": "", "imei": self.hwnd_title, "device": "windows",
+                                                 "model": "", "nick": nick or self.hwnd_title, "width": width,
+                                                 "height": height,
+                                                 "hwnd": nick, "capture": "windows",
                                                  "connected": self.hwnd is not None and self.hwnd.exists,
                                                  "resolution": f"{width}x{height}"}
         communicate.adb_devices.emit()
@@ -114,41 +123,77 @@ class DeviceManager:
         preferred = self.device_dict.get(imei)
         if preferred is None and len(self.device_dict) > 0:
             preferred = next(iter(self.device_dict.values()))
-        if preferred is not None:
             imei = preferred['imei']
-            preferred['preferred'] = True
-            if preferred['method'] == 'windows':
+        if self.config.get("preferred") != imei:
+            self.config["preferred"] = imei
+            self.config.save_file()
+        logger.debug(f'preferred device: {preferred}')
+
+    def get_preferred_device(self):
+        imei = self.config.get("preferred")
+        preferred = self.device_dict.get(imei)
+        if preferred is None:
+            raise ValueError(f"preferred device {imei} is none")
+        return preferred
+
+    def set_hwnd_name(self, hwnd_name):
+        preferred = self.get_preferred_device()
+        if preferred.get("hwnd") != hwnd_name:
+            preferred['hwnd'] = hwnd_name
+            self.config.save_file()
+
+    def set_capture(self, capture):
+        preferred = self.get_preferred_device()
+        preferred['capture'] = capture
+        self.config.save_file()
+
+    def get_hwnd_name(self):
+        preferred = self.get_preferred_device()
+        return preferred.get('hwnd')
+
+    def start(self):
+        preferred = self.get_preferred_device()
+        if preferred['device'] == 'windows':
+            if not isinstance(self.capture_method, WindowsGraphicsCaptureMethod):
+                if self.capture_method is not None:
+                    self.capture_method.close()
+                self.capture_method = WindowsGraphicsCaptureMethod(self.hwnd)
+                self.interaction = Win32Interaction(self.capture_method)
+            self.capture_method.hwnd_window = self.hwnd
+        else:
+            for adb_device in self.adb.device_list():
+                if adb_device.serial == preferred.get('address'):
+                    self._device = adb_device
+            hwnd_name = preferred.get('hwnd')
+            width = preferred.get('width', 0)
+            height = preferred.get('height', 0)
+            if preferred.get('capture') == "windows":
+                if not hwnd_name:
+                    raise ValueError(f"adb device preferred hwnd is none, capture is windows {preferred}")
+                if self.hwnd is None:
+                    self.hwnd = HwndWindow(hwnd_name, self.exit_event, width, height)
                 if not isinstance(self.capture_method, WindowsGraphicsCaptureMethod):
                     if self.capture_method is not None:
                         self.capture_method.close()
                     self.capture_method = WindowsGraphicsCaptureMethod(self.hwnd)
-                    self.interaction = Win32Interaction(self.capture_method)
+                self.hwnd.title = hwnd_name
                 self.capture_method.hwnd_window = self.hwnd
-            else:
-                for adb_device in self.adb.device_list():
-                    if adb_device.serial == preferred.get('address'):
-                        self._device = adb_device
-                if not isinstance(self.capture_method, ADBCaptureMethod):
-                    if self.capture_method is not None:
-                        self.capture_method.close()
-                    self.capture_method = ADBCaptureMethod(self)
-                    self.interaction = ADBBaseInteraction(self, self.capture_method)
-        self.config["preferred"] = imei
-        self.config.save_file()
-        logger.debug(f'preferred device: {preferred}')
-
-    def start(self):
-        if isinstance(self.capture_method, ADBCaptureMethod):
+            elif not isinstance(self.capture_method, ADBCaptureMethod):
+                if self.capture_method is not None:
+                    self.capture_method.close()
+                self.capture_method = ADBCaptureMethod(self._device, self.exit_event, width=width,
+                                                       height=height)
+            self.interaction = ADBBaseInteraction(self, self.capture_method)
             if self.debug:
-                if self.hwnd is not None:
-                    self.hwnd.update_title_re("ok_debug")
-                    self.hwnd.update_frame_size(self.capture_method.width, self.capture_method.height)
-                else:
-                    self.hwnd = HwndWindow("ok_debug", self.exit_event, self.capture_method.width,
-                                           self.capture_method.height)
-            else:
+                if self.hwnd is None and hwnd_name:
+                    if self.hwnd is None:
+                        self.hwnd = HwndWindow(hwnd_name, self.exit_event, width, height)
+                self.hwnd.title = hwnd_name
+            elif self.hwnd is not None:
                 self.hwnd.stop()
                 self.hwnd = None
+            if self.hwnd:
+                self.hwnd.update_frame_size(width, height)
 
     @property
     def device(self):
@@ -156,15 +201,15 @@ class DeviceManager:
 
     @property
     def width(self):
-        if self._size[0] == 0:
-            self.screencap()
-        return self._size[0]
+        if self.capture_method is not None:
+            return self.capture_method.width
+        return 0
 
     @property
     def height(self):
-        if self._size[1] == 0:
-            self.screencap()
-        return self._size[1]
+        if self.capture_method is not None:
+            return self.capture_method.height
+        return 0
 
     def update_device_list(self):
         pass
@@ -176,25 +221,3 @@ class DeviceManager:
                 return device.shell(*args, **kwargs)
             except Exception as e:
                 logger.error(f"shell_wrapper error occurred: {e}")
-
-    def screencap(self):
-        if self.exit_event.is_set():
-            return None
-        self._size = (0, 0)
-        frame = do_screencap(self.device)
-        if frame is not None:
-            height, width, _ = frame.shape
-            self._size = (width, height)
-        return frame
-
-
-def do_screencap(device) -> np.ndarray | None:
-    if device is not None:
-        png_bytes = device.shell("screencap -p", encoding=None)
-        if png_bytes is not None and len(png_bytes) > 0:
-            image_data = np.frombuffer(png_bytes, dtype=np.uint8)
-            image = cv2.imdecode(image_data, cv2.IMREAD_COLOR)
-            if image is not None:
-                return image
-            else:
-                logger.error(f"Screencap image decode error, probably disconnected")

{ok-script-0.0.9 → ok-script-0.0.10}/ok/capture/windows/WindowsGraphicsCaptureMethod.py

@@ -162,8 +162,10 @@ class WindowsGraphicsCaptureMethod(BaseCaptureMethod):
             self.session.Close()  # E_UNEXPECTED ???
             self.session = None
         self.item = None
-        self.rtdevice.Release()
-        self.dxdevice.Release()
+        if self.rtdevice:
+            self.rtdevice.Release()
+        if self.dxdevice:
+            self.dxdevice.Release()
         if self.cputex:
             self.cputex.Release()
 

{ok-script-0.0.9 → ok-script-0.0.10}/ok/feature/Box.py

@@ -6,10 +6,10 @@ import re
 class Box:
     def __init__(self, x: int, y: int, width: int, height: int, confidence: float = 1, name=None) -> None:
         self.name = name
-        self.x = x
-        self.y = y
-        self.width = width
-        self.height = height
+        self.x = int(x)
+        self.y = int(y)
+        self.width = int(width)
+        self.height = int(height)
         self.confidence = confidence
 
     def __eq__(self, other):

{ok-script-0.0.9 → ok-script-0.0.10}/ok/feature/FeatureSet.py

@@ -184,7 +184,7 @@ class FeatureSet:
             # cv2.imwrite("images/test.jpg", mat)
 
         result = sort_boxes(boxes)
-        communicate.draw_box.emit(category_name, result, "red")
+        communicate.emit_draw_box(category_name, result, "red")
         return result
 
 

{ok-script-0.0.9 → ok-script-0.0.10}/ok/gui/App.py

@@ -18,20 +18,21 @@ logger = get_logger(__name__)
 
 
 class App:
-    def __init__(self, icon=None, overlay=False, title="AutoUI", tasks=None, about=None,
+    def __init__(self, config,
                  exit_event=None):
         super().__init__()
+        self.config = config
         self.app = QApplication(sys.argv)
         self.app.setStyle(QStyleFactory.create("Fusion"))
-        self.tasks = tasks
-        self.about = about
-        self.title = title
-        self.overlay = overlay
+        self.tasks = self.config['tasks']
+        self.about = self.config.get('about')
+        self.title = self.config.get('gui_title')
+        self.overlay = self.config.get('debug')
         self.loading_window = None
         self.overlay_window = None
         self.main_window = None
         self.exit_event = exit_event
-        self.icon = QIcon(icon or ":/icon/icon.ico")
+        self.icon = QIcon(self.config.get('gui_icon') or ":/icon/icon.ico")
         self.tray = QSystemTrayIcon(self.icon)
 
         locale = QLocale.system().name()
@@ -64,7 +65,7 @@ class App:
 
     def show_loading(self):
         self.loading_window = LoadingWindow(self, self.exit_event)
-        size = self.size_relative_to_screen(width=0.4, height=0.4)
+        size = self.size_relative_to_screen(width=0.6, height=0.4)
         self.loading_window.resize(size)
         self.loading_window.setMinimumSize(size)
         self.center_window(self.loading_window)

{ok-script-0.0.9 → ok-script-0.0.10}/ok/gui/Communicate.py

@@ -3,12 +3,11 @@ from PySide6.QtCore import Signal, QObject
 
 
 class Communicate(QObject):
-    frame = Signal(np.ndarray)
     log = Signal(int, str)
     fps = Signal(int)
     frame_time = Signal(int)
     scene = Signal(str)
-    draw_box = Signal(str, object, str)
+    draw_box = Signal(str, object, str, np.ndarray)
     tasks = Signal()
     task_info = Signal()
     window = Signal(bool, int, int, int, int, int, int, float)
@@ -16,7 +15,12 @@ class Communicate(QObject):
     init = Signal(bool, str)
     notification = Signal(str, str)
     executor_paused: Signal = Signal(bool)
+    screenshot = Signal(np.ndarray, str)
+    update_overlay = Signal()
     adb_devices: Signal = Signal()
 
+    def emit_draw_box(self, key: str = None, boxes=None, color=None, frame=None):
+        self.draw_box.emit(key, boxes, color, frame)
+
 
 communicate = Communicate()

{ok-script-0.0.9 → ok-script-0.0.10}/ok/gui/__init__.py

@@ -1,5 +1,7 @@
+from ok.OK import OK
 from ok.capture.adb.DeviceManager import DeviceManager
 
+ok: OK | None = None
 executor = None
 app = None
 device_manager: DeviceManager | None = None

ok-script-0.0.10/ok/gui/debug/DebugTab.py

@@ -0,0 +1,31 @@
+from PySide6.QtWidgets import QWidget, QHBoxLayout, QSizePolicy
+
+from ok.gui.debug.InfoWidget import InfoWidget
+from ok.gui.debug.LoggerWidget import LoggerWidget
+
+
+class DebugTab(QWidget):
+    def __init__(self):
+        super().__init__()
+
+        self.mainLayout = QHBoxLayout()
+        self.setLayout(self.mainLayout)
+
+        self.info_widget = InfoWidget()
+        self.info_widget.setFixedWidth(200)
+        # self.frame_widget = FrameWidget(True)
+        self.mainLayout.addWidget(self.info_widget)
+        # self.topSplitter.addWidget(self.frame_widget)
+
+        # Bottom row setup
+        self.logger = LoggerWidget()
+
+        self.mainLayout.addWidget(self.logger)
+
+        p1 = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
+        p1.setHorizontalStretch(1)  # 33% of the layout
+        self.info_widget.setSizePolicy(p1)
+
+        p2 = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
+        p2.setHorizontalStretch(2)  # 67% of the layout
+        self.logger.setSizePolicy(p2)

ok-script-0.0.10/ok/gui/debug/FrameWidget.py

@@ -0,0 +1,58 @@
+import os
+
+from PySide6.QtCore import Qt
+from PySide6.QtGui import QPainter, QColor, QPen
+from PySide6.QtWidgets import QWidget
+
+import ok.gui
+from ok.gui.Communicate import communicate
+from ok.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class FrameWidget(QWidget):
+    def __init__(self):
+        super(FrameWidget, self).__init__()
+
+        self._visible = True
+        communicate.update_overlay.connect(self.update)
+
+    def frame_ratio(self):
+        if ok.gui.device_manager.width == 0:
+            return 1
+        return self.width() / ok.gui.device_manager.width
+
+    def paintEvent(self, event):
+        if not self._visible:
+            return
+        painter = QPainter(self)
+        self.paint_border(painter)
+        self.paint_boxes(painter)
+
+    def paint_boxes(self, painter):
+        pen = QPen()  # Set the brush to red color
+        pen.setWidth(2)  # Set the width of the pen (border thickness)
+        painter.setPen(pen)  # Apply the pen to the painter
+        painter.setBrush(Qt.NoBrush)  # Ensure no fill
+
+        frame_ratio = self.frame_ratio()
+        for key, value in ok.gui.ok.screenshot.ui_dict.items():
+            boxes = value[0]
+            pen.setColor(value[2])
+            painter.setPen(pen)
+            for box in boxes:
+                width = box.width * frame_ratio
+                height = box.height * frame_ratio
+                x = box.x * frame_ratio
+                y = box.y * frame_ratio
+                painter.drawRect(x, y, width, height)
+                painter.drawText(x, y + height + 12, f"{box.name or key}_{round(box.confidence * 100)}")
+
+    def paint_border(self, painter):
+        pen = QPen(QColor(255, 0, 0, 255))  # Solid red color for the border
+        pen.setWidth(1)  # Set the border width
+        painter.setPen(pen)
+
+        # Draw the border around the widget
+        painter.drawRect(0, 0, self.width() - 1, self.height() - 1)

{ok-script-0.0.9 → ok-script-0.0.10}/ok/gui/debug/LoggerWidget.py

@@ -47,6 +47,7 @@ class LoggerWidget(QWidget):
         # Log display
         self.log_display = QTextEdit()
         self.log_display.setReadOnly(True)
+        self.log_display.setStyleSheet("line-height:120%;")
 
         # Add widgets to layout
         self.layout.addLayout(controls_layout)