ok-script 0.0.67__tar.gz → 0.0.68__tar.gz

{ok-script-0.0.67 → ok-script-0.0.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ok-script
-Version: 0.0.67
+Version: 0.0.68
 Summary: Automation with Computer Vision for Python
 Home-page: https://github.com/ok-oldking/ok-script
 Author: ok-oldking

{ok-script-0.0.67 → ok-script-0.0.68}/ok/capture/HwndWindow.py

@@ -50,8 +50,7 @@ class HwndWindow:
         self.title = title
         self.stop_event = threading.Event()
         self.visible = False
-        self.update_frame_size(frame_width, frame_height)
-        self.do_update_window_size()
+        self.update_window(title, exe_name, frame_width, frame_height)
         self.thread = threading.Thread(target=self.update_window_size, name="update_window_size")
         self.thread.start()
 
@@ -68,14 +67,21 @@ class HwndWindow:
     def stop(self):
         self.stop_event.set()
 
+    def update_window(self, title, exe_name, frame_width, frame_height):
+        self.title = title
+        self.exe_name = exe_name
+        self.update_frame_size(frame_width, frame_height)
+
     def update_frame_size(self, width, height):
         logger.debug(f"update_frame_size:{self.frame_width}x{self.frame_height} to {width}x{height}")
         if width != self.frame_width or height != self.frame_height:
+            self.hwnd = None
             self.frame_width = width
             self.frame_height = height
             if width > 0 and height > 0:
                 self.frame_aspect_ratio = width / height
                 logger.debug(f"HwndWindow: frame ratio:{self.frame_aspect_ratio} width: {width}, height: {height}")
+            self.do_update_window_size()
 
     def update_window_size(self):
         while not self.app_exit_event.is_set() and not self.stop_event.is_set():
@@ -194,7 +200,7 @@ def find_hwnd(title, exe_name, frame_aspect_ratio=0):
             y_offset = ratio_match[5] - biggest[5]
             real_width = ratio_match[2]
             real_height = ratio_match[3]
-        logger.debug(f'find_hwnd {results} {biggest} {x_offset, y_offset, real_width, real_height}')
+        logger.debug(f'find_hwnd {results} {biggest} {ratio_match} {x_offset, y_offset, real_width, real_height}')
         return biggest[6], biggest[0], biggest[1], x_offset, y_offset, real_width, real_height,
     else:
         return None, None, None, 0, 0, 0, 0

{ok-script-0.0.67 → ok-script-0.0.68}/ok/capture/adb/DeviceManager.py

@@ -204,8 +204,7 @@ class DeviceManager:
         if self.hwnd is None:
             self.hwnd = HwndWindow(self.exit_event, title, exe, frame_width, frame_height)
         else:
-            self.hwnd.update_frame_size(frame_width, frame_height)
-            self.hwnd.update_title_and_exe(title, exe)
+            self.hwnd.update_window(title, exe, frame_width, frame_height)
 
     def use_windows_capture(self, override_config=None, require_bg=False, use_bit_blt_only=False):
         if not override_config:

{ok-script-0.0.67 → ok-script-0.0.68}/ok/feature/Box.py

@@ -250,3 +250,13 @@ def boxes_to_map_by_list_of_names(boxes: list[Box], name_list: list[str]) -> dic
                 box_dict[box.name] = box
         if len(box_dict) == len(name_list):
             return box_dict
+
+
+def relative_box(frame_width, frame_height, x, y, to_x=1, to_y=1, width=0, height=0, name=None):
+    if width == 0:
+        width = to_x - x
+    if height == 0:
+        height = to_y - y
+    return Box(int(x * frame_width), int(y * frame_height),
+               int(width * frame_width), int(height * frame_height),
+               name=name)

{ok-script-0.0.67 → ok-script-0.0.68}/ok/feature/FeatureSet.py

@@ -143,7 +143,8 @@ class FeatureSet:
             return boxes[0]
 
     def find_feature(self, mat: np.ndarray, category_name: str, horizontal_variance: float = 0,
-                     vertical_variance: float = 0, threshold: float = 0, use_gray_scale: bool = False) -> List[Box]:
+                     vertical_variance: float = 0, threshold: float = 0, use_gray_scale: bool = False, x=-1, y=-1,
+                     to_x=-1, to_y=-1, width=-1, height=-1) -> List[Box]:
         """
         Find a feature within a given variance.
 
@@ -168,15 +169,24 @@ class FeatureSet:
             vertical_variance = self.default_vertical_variance
         if category_name not in self.feature_dict:
             raise ValueError(f"FeatureSet: {category_name} not found in featureDict")
-
         feature = self.feature_dict[category_name]
         feature_width, feature_height = feature.width, feature.height
-
-        # Define search area using variance
-        search_x1 = max(0, int(feature.x - self.width * horizontal_variance))
-        search_y1 = max(0, int(feature.y - self.height * vertical_variance))
-        search_x2 = min(self.width, int(feature.x + feature_width + self.width * horizontal_variance))
-        search_y2 = min(self.height, int(feature.y + feature_height + self.height * vertical_variance))
+        if x != -1 and y != -1:
+            frame_height, frame_width, *_ = mat.shape
+            if width == -1:
+                width = to_x - x
+            if height == -1:
+                height = to_y - y
+            search_x1 = int(x * frame_width)
+            search_y1 = int(y * frame_height)
+            search_x2 = int((x + width) * frame_width)
+            search_y2 = int((y + height) * frame_height)
+        else:
+            # Define search area using variance
+            search_x1 = max(0, int(feature.x - self.width * horizontal_variance))
+            search_y1 = max(0, int(feature.y - self.height * vertical_variance))
+            search_x2 = min(self.width, int(feature.x + feature_width + self.width * horizontal_variance))
+            search_y2 = min(self.height, int(feature.y + feature_height + self.height * vertical_variance))
 
         search_area = mat[search_y1:search_y2, search_x1:search_x2, :3]
         # Crop the search area from the image

{ok-script-0.0.67 → ok-script-0.0.68}/ok/feature/FindFeature.py

@@ -11,11 +11,11 @@ class FindFeature:
         self.executor = None
 
     def find_feature(self, feature_name, horizontal_variance=0, vertical_variance=0, threshold=0,
-                     use_gray_scale=False) -> List[Box]:
+                     use_gray_scale=False, x=-1, y=-1, to_x=-1, to_y=-1, width=-1, height=-1) -> List[Box]:
         if self.executor.frame is None:
             return list()
         return self.feature_set.find_feature(self.executor.frame, feature_name, horizontal_variance, vertical_variance,
-                                             threshold, use_gray_scale)
+                                             threshold, use_gray_scale, x, y, to_x, to_y, width, height)
 
     def get_box_by_name(self, name):
         return self.feature_set.get_box_by_name(name)
@@ -33,9 +33,9 @@ class FindFeature:
             setattr(self, feature, result)
         return ret
 
-    def wait_and_click(self, feature, horizontal_variance=0, vertical_variance=0, threshold=0, relative_x=0.5,
-                       relative_y=0.5,
-                       time_out=0, pre_action=None, post_action=None):
+    def wait_click_feature(self, feature, horizontal_variance=0, vertical_variance=0, threshold=0, relative_x=0.5,
+                           relative_y=0.5,
+                           time_out=0, pre_action=None, post_action=None):
         box = self.wait_until(lambda: self.find_one(feature, horizontal_variance, vertical_variance, threshold),
                               time_out,
                               pre_action,

{ok-script-0.0.67 → ok-script-0.0.68}/ok/gui/debug/DebugTab.py

@@ -12,6 +12,7 @@ from ok.gui.util.Alert import alert_info, alert_error
 from ok.gui.widget.Tab import Tab
 from ok.interaction.DoNothingInteraction import DoNothingInteraction
 from ok.logging.Logger import get_logger, exception_to_str
+from ok.ocr.OCR import OCR
 from ok.util.Handler import Handler
 
 logger = get_logger(__name__)
@@ -44,6 +45,10 @@ class DebugTab(Tab):
         capture_button.clicked.connect(lambda: self.handler.post(self.capture))
         layout.addWidget(capture_button)
 
+        ocr_button = PushButton("OCR")
+        ocr_button.clicked.connect(lambda: self.handler.post(self.ocr))
+        layout.addWidget(ocr_button)
+
         open_log_folder = PushButton(self.tr("Open Logs"))
 
         call_task_widget = QWidget()
@@ -116,6 +121,25 @@ class DebugTab(Tab):
         self.config['target_function'] = func_name
         alert_info(self.tr(f"call success: {result}"))
 
+    def ocr(self):
+        if not ok.gui.executor.ocr:
+            alert_error(self.tr('No OCR configured'))
+            return
+        if ok.gui.device_manager.capture_method is not None:
+            logger.info(f'ok.gui.device_manager.capture_method {ok.gui.device_manager.capture_method}')
+            capture = str(ok.gui.device_manager.capture_method)
+            frame = ok.gui.device_manager.capture_method.do_get_frame()
+            if frame is not None:
+                ocr = OCR()
+                ocr.executor = ok.gui.executor
+                result = ocr.ocr(frame=frame)
+                self.update_result_text.emit(str(result))
+                alert_info(self.tr(f"OCR success: {result}"))
+            else:
+                alert_error(self.tr('Capture returned None'))
+        else:
+            alert_error(self.tr('No Capture Available or Selected'))
+
     def task_changed(self, text):
         self.config['target_task'] = text
         task = ok.gui.executor.get_task_by_class_name(text)

{ok-script-0.0.67 → ok-script-0.0.68}/ok/gui/debug/FrameWidget.py

@@ -1,7 +1,8 @@
 import os
 
-from PySide6.QtCore import Qt
-from PySide6.QtGui import QPainter, QColor, QPen
+import win32api
+from PySide6.QtCore import Qt, QPoint, QTimer
+from PySide6.QtGui import QPainter, QColor, QPen, QFont, QGuiApplication
 from PySide6.QtWidgets import QWidget
 
 import ok.gui
@@ -14,10 +15,27 @@ logger = get_logger(__name__)
 class FrameWidget(QWidget):
     def __init__(self):
         super(FrameWidget, self).__init__()
-
         self._visible = True
+        self._mouse_position = QPoint(0, 0)
+        self.setMouseTracking(True)
+        communicate.update_overlay.connect(self.update)
+        # Start a timer to update mouse position using Windows API
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.update_mouse_position)
+        self.timer.start(1000)  # Update every 50 milliseconds
+        self.mouse_font = QFont()
+        self.mouse_font.setPointSize(10)  # Adjust the size as needed
+        screen = QGuiApplication.primaryScreen()
+        self.scaling = screen.devicePixelRatio()
         communicate.update_overlay.connect(self.update)
 
+    def update_mouse_position(self):
+        x, y = win32api.GetCursorPos()
+        relative = self.mapFromGlobal(QPoint(x / self.scaling, y / self.scaling))
+        if self._mouse_position != relative and relative.x() > 0 and relative.y() > 0 and self._visible:
+            self._mouse_position = relative
+            self.update()
+
     def frame_ratio(self):
         if ok.gui.device_manager.width == 0:
             return 1
@@ -29,6 +47,7 @@ class FrameWidget(QWidget):
         painter = QPainter(self)
         self.paint_border(painter)
         self.paint_boxes(painter)
+        self.paint_mouse_position(painter)
 
     def paint_boxes(self, painter):
         pen = QPen()  # Set the brush to red color
@@ -53,6 +72,19 @@ class FrameWidget(QWidget):
         pen = QPen(QColor(255, 0, 0, 255))  # Solid red color for the border
         pen.setWidth(1)  # Set the border width
         painter.setPen(pen)
-
         # Draw the border around the widget
         painter.drawRect(0, 0, self.width() - 1, self.height() - 1)
+
+    def paint_mouse_position(self, painter):
+        x_percent = self._mouse_position.x() / self.width()
+        y_percent = self._mouse_position.y() / self.height()
+        x, y = self._mouse_position.x() * 2, self._mouse_position.y() * 2
+        text = f"({x}, {y}, {x_percent:.2f}, {y_percent:.2f})"
+        painter.setFont(self.mouse_font)
+
+        painter.setPen(QPen(QColor(255, 0, 0, 255), 1))
+        painter.drawText(20, 20, text)
+        #
+        # # Draw the black text
+        # painter.setPen(QPen(QColor(0, 0, 0), 2))
+        # painter.drawText(10, 10, text)

{ok-script-0.0.67 → ok-script-0.0.68}/ok/gui/overlay/OverlayWindow.py

@@ -15,15 +15,20 @@ class Communicate(QObject):
 class OverlayWindow(FrameWidget):
     def __init__(self, hwnd_window: HwndWindow):
         super().__init__()
-        self.setWindowFlag(Qt.WindowStaysOnTopHint, True)
-        self.setWindowFlag(Qt.FramelessWindowHint)  # Make the window frameless
+        # Set translucent background
         self.setAttribute(Qt.WA_TranslucentBackground)
-        self.setWindowFlag(Qt.WindowTransparentForInput, True)
-        self.setWindowFlag(Qt.Tool)
+
+        # Ensure mouse events are received properly
+        self.setAttribute(Qt.WA_OpaquePaintEvent)
+
+        # Set window flag to handle mouse events properly with translucent background
+        self.setWindowFlags(Qt.FramelessWindowHint | Qt.WindowStaysOnTopHint | Qt.Tool | Qt.WindowTransparentForInput)
         communicate.window.connect(self.update_overlay)
         self.update_overlay(hwnd_window.visible, hwnd_window.x, hwnd_window.y, hwnd_window.border,
                             hwnd_window.title_height, hwnd_window.width, hwnd_window.height, hwnd_window.scaling)
 
+        self.setMouseTracking(True)
+
     def update_overlay(self, visible, x, y, border, title_height, width, height, scaling):
         logger.debug(f'update_overlay: {visible}, {x}, {y}, {border}, {title_height} {width}, {height}, {scaling}')
         if visible: