ok-script 0.0.67__tar.gz → 0.0.69__tar.gz

{ok-script-0.0.67 → ok-script-0.0.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ok-script
-Version: 0.0.67
+Version: 0.0.69
 Summary: Automation with Computer Vision for Python
 Home-page: https://github.com/ok-oldking/ok-script
 Author: ok-oldking

{ok-script-0.0.67 → ok-script-0.0.69}/ok/capture/HwndWindow.py

@@ -50,8 +50,7 @@ class HwndWindow:
         self.title = title
         self.stop_event = threading.Event()
         self.visible = False
-        self.update_frame_size(frame_width, frame_height)
-        self.do_update_window_size()
+        self.update_window(title, exe_name, frame_width, frame_height)
         self.thread = threading.Thread(target=self.update_window_size, name="update_window_size")
         self.thread.start()
 
@@ -68,14 +67,21 @@ class HwndWindow:
     def stop(self):
         self.stop_event.set()
 
+    def update_window(self, title, exe_name, frame_width, frame_height):
+        self.title = title
+        self.exe_name = exe_name
+        self.update_frame_size(frame_width, frame_height)
+
     def update_frame_size(self, width, height):
         logger.debug(f"update_frame_size:{self.frame_width}x{self.frame_height} to {width}x{height}")
         if width != self.frame_width or height != self.frame_height:
+            self.hwnd = None
             self.frame_width = width
             self.frame_height = height
             if width > 0 and height > 0:
                 self.frame_aspect_ratio = width / height
-                logger.debug(f"HwndWindow: frame ratio:{self.frame_aspect_ratio} width: {width}, height: {height}")
+                logger.debug(f"HwndWindow: frame ratio: width: {width}, height: {height}")
+            self.do_update_window_size()
 
     def update_window_size(self):
         while not self.app_exit_event.is_set() and not self.stop_event.is_set():
@@ -95,7 +101,7 @@ class HwndWindow:
                 name, self.hwnd, self.exe_full_path, self.real_x_offset, self.real_y_offset, self.real_width, self.real_height = find_hwnd(
                     self.title,
                     self.exe_name,
-                    self.frame_aspect_ratio)
+                    self.frame_width, self.frame_height)
                 self.exists = self.hwnd is not None
             if self.hwnd is not None:
                 self.exists = win32gui.IsWindow(self.hwnd)
@@ -147,10 +153,11 @@ class HwndWindow:
         return f"title:{self.title}_{self.exe_name}_{self.width}x{self.height}_{self.hwnd}_{self.exists}_{self.visible}"
 
 
-def find_hwnd(title, exe_name, frame_aspect_ratio=0):
+def find_hwnd(title, exe_name, frame_width, frame_height):
     results = []
     if exe_name is None and title is None:
         return None, None, None, 0, 0, 0, 0
+    frame_aspect_ratio = frame_width / frame_height if frame_height != 0 else 0
 
     def callback(hwnd, lParam):
         if win32gui.IsWindowVisible(hwnd) and win32gui.IsWindowEnabled(hwnd):
@@ -173,7 +180,6 @@ def find_hwnd(title, exe_name, frame_aspect_ratio=0):
 
     win32gui.EnumWindows(callback, None)
     biggest = None
-    ratio_match = None
     if len(results) > 0:
         for result in results:
             from ok.capture.windows.BitBltCaptureMethod import bit_blt_test_hwnd
@@ -189,17 +195,46 @@ def find_hwnd(title, exe_name, frame_aspect_ratio=0):
         x_offset = 0
         y_offset = 0
         real_width, real_height = biggest[2], biggest[3]
-        if ratio_match is not None and ratio_match[4] - biggest[4] >= 0 and ratio_match[5] - biggest[5] >= 0:
-            x_offset = ratio_match[4] - biggest[4]
-            y_offset = ratio_match[5] - biggest[5]
-            real_width = ratio_match[2]
-            real_height = ratio_match[3]
+        if biggest is not None and frame_aspect_ratio != 0:
+            matching_child = enum_child_windows(biggest, frame_aspect_ratio)
+            if matching_child is not None:
+                x_offset, y_offset, real_width, real_height = matching_child
         logger.debug(f'find_hwnd {results} {biggest} {x_offset, y_offset, real_width, real_height}')
         return biggest[6], biggest[0], biggest[1], x_offset, y_offset, real_width, real_height,
     else:
         return None, None, None, 0, 0, 0, 0
 
 
+def enum_child_windows(biggest, frame_aspect_ratio):
+    ratio_match = []
+    """
+    A function to enumerate all child windows of the given parent window handle
+    and print their handle and window title.
+    """
+
+    def child_callback(hwnd, _):
+        visible = win32gui.IsWindowVisible(hwnd)
+        parent = win32gui.GetParent(hwnd)
+        rect = win32gui.GetWindowRect(hwnd)
+        logger.debug(f'find_hwnd child_callback {visible} {biggest[0]} {parent} {rect}')
+        if visible and parent == biggest[0]:
+            real_width = rect[2] - rect[0]
+            real_height = rect[3] - rect[1]
+            ratio = real_width / real_height
+            difference = abs(ratio - frame_aspect_ratio)
+            support = difference <= 0.01 * frame_aspect_ratio
+            percent = (real_width * real_height) / (biggest[2] * biggest[3])
+            if support and percent >= 0.7:
+                x_offset = rect[0] - biggest[4]
+                y_offset = rect[1] - biggest[5]
+                ratio_match.append((x_offset, y_offset, real_width, real_height))
+        return True
+
+    win32gui.EnumChildWindows(biggest[0], child_callback, None)
+    if len(ratio_match) > 0:
+        return ratio_match[0]
+
+
 def get_exe_by_hwnd(hwnd):
     # Get the process ID associated with the window
     _, pid = win32process.GetWindowThreadProcessId(hwnd)
@@ -230,10 +265,28 @@ def enum_windows(emulator_path=None):
     return windows
 
 
+#
+# def enum_child_windows(parent_handle):
+#     """
+#     A function to enumerate all child windows of the given parent window handle
+#     and print their handle and window title.
+#     """
+#
+#     def callback(hwnd, _):
+#         if win32gui.IsWindowVisible(hwnd) and win32gui.GetParent(hwnd) == parent_handle:
+#             rect = win32gui.GetWindowRect(hwnd)
+#             width = rect[2] - rect[0]
+#             height = rect[3] - rect[1]
+#             print(f"Child HWND: {hwnd} - Window Text: {win32gui.GetWindowText(hwnd)} {rect} {width} {height}")
+#         return True
+#
+#     win32gui.EnumChildWindows(parent_handle, callback, None)
+
+
 if __name__ == '__main__':
     print(find_hwnd("MuMu模拟器12", None))
-    hwnd_window = HwndWindow(threading.Event(), None, 'D:\\MuMuPlayer-12.0\\shell\\MuMuPlayer.exe', 16, 9)
-    from ok.capture.windows.BitBltCaptureMethod import BitBltCaptureMethod
-
-    method = BitBltCaptureMethod(hwnd_window)
-    method.get_frame()
+    # hwnd_window = HwndWindow(threading.Event(), None, 'D:\\MuMuPlayer-12.0\\shell\\MuMuPlayer.exe', 16, 9)
+    # from ok.capture.windows.BitBltCaptureMethod import BitBltCaptureMethod
+    #
+    # method = BitBltCaptureMethod(hwnd_window)
+    # method.get_frame()

{ok-script-0.0.67 → ok-script-0.0.69}/ok/capture/adb/DeviceManager.py

@@ -25,6 +25,7 @@ class DeviceManager:
         self.interaction = None
         self.device_dict = {}
         self.exit_event = exit_event
+        self.resolution_dict = {}
         if windows_capture_config is not None:
             self.hwnd = HwndWindow(exit_event, windows_capture_config.get('title'),
                                    windows_capture_config.get('exe'))
@@ -73,7 +74,7 @@ class DeviceManager:
     def update_pc_device(self):
         if self.windows_capture_config is not None:
             name, hwnd, full_path, x, y, width, height = find_hwnd(self.windows_capture_config.get('title'),
-                                                                   self.windows_capture_config.get('exe'))
+                                                                   self.windows_capture_config.get('exe'), 0, 0)
             nick = name or self.windows_capture_config.get('exe')
             pc_device = {"address": "", "imei": 'pc', "device": "windows",
                          "model": "", "nick": nick, "width": width,
@@ -98,7 +99,7 @@ class DeviceManager:
 
         if self.exit_event.is_set():
             return
-        self.start()
+        self.do_start()
 
         logger.debug(f'refresh {self.device_dict}')
 
@@ -112,10 +113,11 @@ class DeviceManager:
                         found = True
                         break
                 if not found:
+                    height, width = self.get_resolution(adb_device)
                     logger.debug(f'refresh_phones found an phone {adb_device}')
                     phone_device = {"address": adb_device.serial, "device": "adb", "connected": True, "imei": imei,
                                     "nick": adb_device.prop.model or imei,
-                                    "resolution": self.get_resolution(adb_device)}
+                                    "resolution": f'{width}x{height}'}
                     self.device_dict[imei] = phone_device
 
     def refresh_emulators(self):
@@ -124,8 +126,9 @@ class DeviceManager:
         installed_emulators = manager.all_emulator_instances
         for emulator in installed_emulators:
             adb_device = self.adb_connect(emulator.serial)
+            width, height = self.get_resolution(adb_device)
             name, hwnd, full_path, x, y, width, height = find_hwnd(None,
-                                                                   emulator.path)
+                                                                   emulator.path, width, height)
             connected = adb_device is not None and name is not None
             emulator_device = {"address": emulator.serial, "device": "adb",
                                "full_path": emulator.path, "connected": connected,
@@ -142,9 +145,12 @@ class DeviceManager:
             device = self.device
         width, height = 0, 0
         if device is not None:
+            if resolution := self.resolution_dict.get(device.serial):
+                return resolution
             frame = do_screencap(device)
             if frame is not None:
                 height, width, _ = frame.shape
+                self.resolution_dict[device.serial] = (width, height)
         return height, width
 
     def set_preferred_device(self, imei=None, index=-1):
@@ -156,7 +162,12 @@ class DeviceManager:
         preferred = self.device_dict.get(imei)
         if preferred is None:
             if len(self.device_dict) > 0:
-                preferred = next(iter(self.device_dict.values()))
+                connected_device = None
+                for device in self.device_dict.values():
+                    if device.get('connected') or connected_device is None:
+                        connected_device = device
+                logger.info(f'first start use first or connected device {connected_device}')
+                preferred = connected_device
                 imei = preferred['imei']
             else:
                 logger.warning(f'no devices')
@@ -204,8 +215,7 @@ class DeviceManager:
         if self.hwnd is None:
             self.hwnd = HwndWindow(self.exit_event, title, exe, frame_width, frame_height)
         else:
-            self.hwnd.update_frame_size(frame_width, frame_height)
-            self.hwnd.update_title_and_exe(title, exe)
+            self.hwnd.update_window(title, exe, frame_width, frame_height)
 
     def use_windows_capture(self, override_config=None, require_bg=False, use_bit_blt_only=False):
         if not override_config:
@@ -238,13 +248,14 @@ class DeviceManager:
             if self.config.get('capture') == "windows":
                 self.ensure_hwnd(None, preferred.get('full_path'), width, height)
                 self.use_windows_capture({'can_bit_blt': True}, require_bg=True, use_bit_blt_only=True)
-            elif not isinstance(self.capture_method, ADBCaptureMethod):
-                logger.debug(f'use adb capture')
-                if self.capture_method is not None:
-                    self.capture_method.close()
-                self.capture_method = ADBCaptureMethod(self, self.exit_event, width=width,
-                                                       height=height)
-                if self.debug:
+            else:
+                if not isinstance(self.capture_method, ADBCaptureMethod):
+                    logger.debug(f'use adb capture')
+                    if self.capture_method is not None:
+                        self.capture_method.close()
+                    self.capture_method = ADBCaptureMethod(self, self.exit_event, width=width,
+                                                           height=height)
+                if self.debug and preferred.get('full_path'):
                     self.ensure_hwnd(None, preferred.get('full_path'), width, height)
                 elif self.hwnd is not None:
                     self.hwnd.stop()

{ok-script-0.0.67 → ok-script-0.0.69}/ok/feature/Box.py

@@ -250,3 +250,13 @@ def boxes_to_map_by_list_of_names(boxes: list[Box], name_list: list[str]) -> dic
                 box_dict[box.name] = box
         if len(box_dict) == len(name_list):
             return box_dict
+
+
+def relative_box(frame_width, frame_height, x, y, to_x=1, to_y=1, width=0, height=0, name=None):
+    if width == 0:
+        width = to_x - x
+    if height == 0:
+        height = to_y - y
+    return Box(int(x * frame_width), int(y * frame_height),
+               int(width * frame_width), int(height * frame_height),
+               name=name)

{ok-script-0.0.67 → ok-script-0.0.69}/ok/feature/FeatureSet.py

@@ -143,7 +143,8 @@ class FeatureSet:
             return boxes[0]
 
     def find_feature(self, mat: np.ndarray, category_name: str, horizontal_variance: float = 0,
-                     vertical_variance: float = 0, threshold: float = 0, use_gray_scale: bool = False) -> List[Box]:
+                     vertical_variance: float = 0, threshold: float = 0, use_gray_scale: bool = False, x=-1, y=-1,
+                     to_x=-1, to_y=-1, width=-1, height=-1) -> List[Box]:
         """
         Find a feature within a given variance.
 
@@ -168,15 +169,24 @@ class FeatureSet:
             vertical_variance = self.default_vertical_variance
         if category_name not in self.feature_dict:
             raise ValueError(f"FeatureSet: {category_name} not found in featureDict")
-
         feature = self.feature_dict[category_name]
         feature_width, feature_height = feature.width, feature.height
-
-        # Define search area using variance
-        search_x1 = max(0, int(feature.x - self.width * horizontal_variance))
-        search_y1 = max(0, int(feature.y - self.height * vertical_variance))
-        search_x2 = min(self.width, int(feature.x + feature_width + self.width * horizontal_variance))
-        search_y2 = min(self.height, int(feature.y + feature_height + self.height * vertical_variance))
+        if x != -1 and y != -1:
+            frame_height, frame_width, *_ = mat.shape
+            if width == -1:
+                width = to_x - x
+            if height == -1:
+                height = to_y - y
+            search_x1 = int(x * frame_width)
+            search_y1 = int(y * frame_height)
+            search_x2 = int((x + width) * frame_width)
+            search_y2 = int((y + height) * frame_height)
+        else:
+            # Define search area using variance
+            search_x1 = max(0, int(feature.x - self.width * horizontal_variance))
+            search_y1 = max(0, int(feature.y - self.height * vertical_variance))
+            search_x2 = min(self.width, int(feature.x + feature_width + self.width * horizontal_variance))
+            search_y2 = min(self.height, int(feature.y + feature_height + self.height * vertical_variance))
 
         search_area = mat[search_y1:search_y2, search_x1:search_x2, :3]
         # Crop the search area from the image

{ok-script-0.0.67 → ok-script-0.0.69}/ok/feature/FindFeature.py

@@ -11,11 +11,11 @@ class FindFeature:
         self.executor = None
 
     def find_feature(self, feature_name, horizontal_variance=0, vertical_variance=0, threshold=0,
-                     use_gray_scale=False) -> List[Box]:
+                     use_gray_scale=False, x=-1, y=-1, to_x=-1, to_y=-1, width=-1, height=-1) -> List[Box]:
         if self.executor.frame is None:
             return list()
         return self.feature_set.find_feature(self.executor.frame, feature_name, horizontal_variance, vertical_variance,
-                                             threshold, use_gray_scale)
+                                             threshold, use_gray_scale, x, y, to_x, to_y, width, height)
 
     def get_box_by_name(self, name):
         return self.feature_set.get_box_by_name(name)
@@ -33,9 +33,9 @@ class FindFeature:
             setattr(self, feature, result)
         return ret
 
-    def wait_and_click(self, feature, horizontal_variance=0, vertical_variance=0, threshold=0, relative_x=0.5,
-                       relative_y=0.5,
-                       time_out=0, pre_action=None, post_action=None):
+    def wait_click_feature(self, feature, horizontal_variance=0, vertical_variance=0, threshold=0, relative_x=0.5,
+                           relative_y=0.5,
+                           time_out=0, pre_action=None, post_action=None):
         box = self.wait_until(lambda: self.find_one(feature, horizontal_variance, vertical_variance, threshold),
                               time_out,
                               pre_action,

{ok-script-0.0.67 → ok-script-0.0.69}/ok/firebase/FireBaseAnalytics.py

@@ -1,5 +1,10 @@
+import ctypes
 import json
+import locale
+import random
+import time
 import uuid
+from urllib.parse import urlencode
 
 import requests
 import wmi
@@ -10,6 +15,8 @@ from ok.util.Handler import Handler
 
 FIREBASE_ENDPOINT = 'https://www.google-analytics.com/mp/collect'
 
+GTAG_ENDPOINT = "https://www.google-analytics.com/g/collect"
+
 logger = get_logger(__name__)
 
 
@@ -20,8 +27,9 @@ class FireBaseAnalytics:
         self.app_config = app_config
         self._config = None
         self._handler = Handler(exit_event, __name__)
-        self._handler.post(self.report_open, 10)
+        self._handler.post(self.send_gtag, 0)
         self._user_properties = None
+        self._fv = 1
 
     @property
     def user_properties(self):
@@ -64,8 +72,10 @@ class FireBaseAnalytics:
             }
         return self._user_properties
 
-    def report_open(self):
-        self.send_event('app_open', {'version': self.app_config.get('version'), 'debug': self.app_config.get('debug')})
+    #
+    # def report_open(self):
+    #     self._handler.post(self.send_gtag, 5)
+    #     # self.send_event('app_open', {'version': self.app_config.get('version'), 'debug': self.app_config.get('debug')})
 
     @property
     def client_id(self):
@@ -73,8 +83,66 @@ class FireBaseAnalytics:
             self._config = Config({'client_id': ''}, self.app_config.get("config_folder") or "config", 'statistics')
         if not self._config.get('client_id'):
             self._config['client_id'] = get_unique_client_id()
+        else:
+            self._fv = 0
         return self._config.get('client_id')
 
+    def send_gtag(self):
+        # Define the parameters in a dictionary
+        params = {
+            'v': '2',
+            'tid': self.measurement_id,
+            'gtm': '45je4630v9186643593za200',
+            '_p': random_number(),
+            # 'gcd': '13l3l3l3l1',
+            'npa': '0',
+            'dma': '0',
+            'tag_exp': '0',
+            'cid': self.client_id,
+            'ul': locale.getdefaultlocale()[0],
+            'sr': get_screen_resolution(),
+            'uaa': 'x86',
+            'uab': '64',
+            # 'uafvl': 'Microsoft Edge;125.0.2535.85|Chromium;125.0.6422.142|Not.A/Brand;24.0.0.0',
+            'uamb': '0',
+            'uam': '',
+            'uap': 'Windows',
+            'uapv': '15.0.0',
+            'uaw': '0',
+            'are': '1',
+            'frm': '0',
+            'pscdl': 'noapi',
+            '_s': '1',
+            'sid': int(time.time()),
+            'sct': '1',
+            'seg': '1',
+            'dl': 'https://ok-script.com/',
+            'dt': 'AppStartGTAG2',
+            'en': 'page_view',
+            '_ee': '1',
+            'ep.version': self.app_config.get('version')
+            # 'tfd': '5133'
+        }
+        if self._fv:
+            params['_fv'] = self._fv
+
+        # Send the GET request
+
+        headers = {
+            'Content-Type': 'application/json',
+            'Origin': 'https://ok-script.com',
+            'Referer': 'https://ok-script.com/'
+        }
+        query_string = urlencode(params)
+
+        response = requests.post(f'{GTAG_ENDPOINT}', data=params)
+
+        logger.debug(f'send gtag {GTAG_ENDPOINT}?{query_string}')
+        if response.status_code == 204:
+            logger.debug(f'Successfully sent gtag')
+        else:
+            logger.debug(f'Failed to send event: {response.status_code} - {response.text}')
+
     def send_event(self, event_name, params=None):
         if params is None:
             params = dict()
@@ -107,7 +175,17 @@ class FireBaseAnalytics:
 
 
 def get_unique_client_id():
-    return str(uuid.uuid4())
+    return f'{random_number()}.{random_number()}'
+
+
+def random_number():
+    return random.randint(0, 2147483647)
+
+
+def get_screen_resolution():
+    user32 = ctypes.windll.user32
+    screensize = user32.GetSystemMetrics(0), user32.GetSystemMetrics(1)
+    return f"{screensize[0]}x{screensize[1]}"
 
 
 if __name__ == '__main__':

{ok-script-0.0.67 → ok-script-0.0.69}/ok/gui/App.py

@@ -8,11 +8,11 @@ from qfluentwidgets import FluentTranslator, qconfig, Theme
 
 import ok
 import ok.gui.resources
-from ok.capture.adb.StartController import StartController
 from ok.firebase.FireBaseAnalytics import FireBaseAnalytics
 from ok.gui.Communicate import communicate
 from ok.gui.MainWindow import MainWindow
 from ok.gui.MessageWindow import MessageWindow
+from ok.gui.StartController import StartController
 from ok.gui.i18n.path import i18n_path
 from ok.gui.overlay.OverlayWindow import OverlayWindow
 from ok.logging.Logger import get_logger

{ok-script-0.0.67/ok/capture/adb → ok-script-0.0.69/ok/gui}/StartController.py

@@ -26,7 +26,7 @@ class StartController(QObject):
 
     def do_start(self, task=None):
         communicate.starting_emulator.emit(False, None, 30)
-        self.device_manager.do_refresh(fast=True)
+        self.device_manager.do_refresh()
 
         device = self.device_manager.get_preferred_device()
 
@@ -37,6 +37,7 @@ class StartController(QObject):
                 execute(path)
                 wait_until = time.time() + 30
                 while not self.exit_event.is_set():
+                    self.device_manager.do_refresh()
                     error = self.check_device_error()
                     if error is None:
                         break

{ok-script-0.0.67 → ok-script-0.0.69}/ok/gui/debug/DebugTab.py

@@ -12,6 +12,7 @@ from ok.gui.util.Alert import alert_info, alert_error
 from ok.gui.widget.Tab import Tab
 from ok.interaction.DoNothingInteraction import DoNothingInteraction
 from ok.logging.Logger import get_logger, exception_to_str
+from ok.ocr.OCR import OCR
 from ok.util.Handler import Handler
 
 logger = get_logger(__name__)
@@ -44,6 +45,10 @@ class DebugTab(Tab):
         capture_button.clicked.connect(lambda: self.handler.post(self.capture))
         layout.addWidget(capture_button)
 
+        ocr_button = PushButton("OCR")
+        ocr_button.clicked.connect(lambda: self.handler.post(self.ocr))
+        layout.addWidget(ocr_button)
+
         open_log_folder = PushButton(self.tr("Open Logs"))
 
         call_task_widget = QWidget()
@@ -116,6 +121,25 @@ class DebugTab(Tab):
         self.config['target_function'] = func_name
         alert_info(self.tr(f"call success: {result}"))
 
+    def ocr(self):
+        if not ok.gui.executor.ocr:
+            alert_error(self.tr('No OCR configured'))
+            return
+        if ok.gui.device_manager.capture_method is not None:
+            logger.info(f'ok.gui.device_manager.capture_method {ok.gui.device_manager.capture_method}')
+            capture = str(ok.gui.device_manager.capture_method)
+            frame = ok.gui.device_manager.capture_method.do_get_frame()
+            if frame is not None:
+                ocr = OCR()
+                ocr.executor = ok.gui.executor
+                result = ocr.ocr(frame=frame)
+                self.update_result_text.emit(str(result))
+                alert_info(self.tr(f"OCR success: {result}"))
+            else:
+                alert_error(self.tr('Capture returned None'))
+        else:
+            alert_error(self.tr('No Capture Available or Selected'))
+
     def task_changed(self, text):
         self.config['target_task'] = text
         task = ok.gui.executor.get_task_by_class_name(text)

ok-script-0.0.69/ok/gui/debug/FrameWidget.py

@@ -0,0 +1,93 @@
+import os
+
+import win32api
+from PySide6.QtCore import Qt, QPoint, QTimer
+from PySide6.QtGui import QPainter, QColor, QPen, QFont, QGuiApplication
+from PySide6.QtWidgets import QWidget
+
+import ok.gui
+from ok.gui.Communicate import communicate
+from ok.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class FrameWidget(QWidget):
+    def __init__(self):
+        super(FrameWidget, self).__init__()
+        self._visible = True
+        self._mouse_position = QPoint(0, 0)
+        self.setMouseTracking(True)
+        communicate.update_overlay.connect(self.update)
+        # Start a timer to update mouse position using Windows API
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self.update_mouse_position)
+        self.timer.start(1000)  # Update every 50 milliseconds
+        self.mouse_font = QFont()
+        self.mouse_font.setPointSize(10)  # Adjust the size as needed
+        screen = QGuiApplication.primaryScreen()
+        self.scaling = screen.devicePixelRatio()
+        communicate.update_overlay.connect(self.update)
+
+    def update_mouse_position(self):
+        try:
+            x, y = win32api.GetCursorPos()
+            relative = self.mapFromGlobal(QPoint(x / self.scaling, y / self.scaling))
+            if self._mouse_position != relative and relative.x() > 0 and relative.y() > 0 and self._visible:
+                self._mouse_position = relative
+                self.update()
+        except Exception as e:
+            logger.warning(f'GetCursorPos exception {e}')
+
+    def frame_ratio(self):
+        if ok.gui.device_manager.width == 0:
+            return 1
+        return self.width() / ok.gui.device_manager.width
+
+    def paintEvent(self, event):
+        if not self._visible:
+            return
+        painter = QPainter(self)
+        self.paint_border(painter)
+        self.paint_boxes(painter)
+        self.paint_mouse_position(painter)
+
+    def paint_boxes(self, painter):
+        pen = QPen()  # Set the brush to red color
+        pen.setWidth(2)  # Set the width of the pen (border thickness)
+        painter.setPen(pen)  # Apply the pen to the painter
+        painter.setBrush(Qt.NoBrush)  # Ensure no fill
+
+        frame_ratio = self.frame_ratio()
+        for key, value in ok.gui.ok.screenshot.ui_dict.items():
+            boxes = value[0]
+            pen.setColor(value[2])
+            painter.setPen(pen)
+            for box in boxes:
+                width = box.width * frame_ratio
+                height = box.height * frame_ratio
+                x = box.x * frame_ratio
+                y = box.y * frame_ratio
+                painter.drawRect(x, y, width, height)
+                painter.drawText(x, y + height + 12, f"{box.name or key}_{round(box.confidence * 100)}")
+
+    def paint_border(self, painter):
+        pen = QPen(QColor(255, 0, 0, 255))  # Solid red color for the border
+        pen.setWidth(1)  # Set the border width
+        painter.setPen(pen)
+        # Draw the border around the widget
+        painter.drawRect(0, 0, self.width() - 1, self.height() - 1)
+
+    def paint_mouse_position(self, painter):
+        x_percent = self._mouse_position.x() / self.width()
+        y_percent = self._mouse_position.y() / self.height()
+        x, y = self._mouse_position.x() * 2, self._mouse_position.y() * 2
+        text = f"({x}, {y}, {x_percent:.2f}, {y_percent:.2f})"
+        painter.setFont(self.mouse_font)
+
+        painter.setPen(QPen(QColor(255, 0, 0, 255), 1))
+        painter.drawText(20, 20, text)
+        #
+        # # Draw the black text
+        # painter.setPen(QPen(QColor(0, 0, 0), 2))
+        # painter.drawText(10, 10, text)