ok-script 0.0.86__tar.gz → 0.0.87__tar.gz

{ok-script-0.0.86 → ok-script-0.0.87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ok-script
-Version: 0.0.86
+Version: 0.0.87
 Summary: Automation with Computer Vision for Python
 Home-page: https://github.com/ok-oldking/ok-script
 Author: ok-oldking

{ok-script-0.0.86 → ok-script-0.0.87}/ok/OK.py

@@ -97,7 +97,7 @@ class OK:
         if template_matching is not None:
             coco_feature_json = self.config.get('template_matching').get('coco_feature_json')
             from ok.feature.FeatureSet import FeatureSet
-            self.feature_set = FeatureSet(coco_feature_json,
+            self.feature_set = FeatureSet(self.debug, coco_feature_json,
                                           default_horizontal_variance=template_matching.get(
                                               'default_horizontal_variance', 0),
                                           default_vertical_variance=template_matching.get('default_vertical_variance',

ok-script-0.0.87/ok/capture/adb/WindowsCaptureFactory.py

@@ -0,0 +1,42 @@
+from ok.logging.Logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def update_capture_method(config, capture_method, hwnd, require_bg=False, use_bit_blt_only=False):
+    try:
+        if config.get('can_bit_blt'):
+            logger.debug(
+                f"try BitBlt method {config} {hwnd} current_type:{type(capture_method)}")
+            from ok.capture.windows.BitBltCaptureMethod import BitBltCaptureMethod
+            target_method = BitBltCaptureMethod
+            capture_method = get_capture(capture_method, target_method, hwnd)
+            if capture_method.test_exclusive_full_screen():
+                return capture_method
+            else:
+                logger.info("test_exclusive_full_screen failed, can't use BitBlt")
+        if use_bit_blt_only:
+            return None
+        from ok.capture.windows.WindowsGraphicsCaptureMethod import windows_graphics_available
+        if windows_graphics_available():
+            from ok.capture.windows.WindowsGraphicsCaptureMethod import WindowsGraphicsCaptureMethod
+            target_method = WindowsGraphicsCaptureMethod
+            capture_method = get_capture(capture_method, target_method, hwnd)
+            return capture_method
+
+        if not require_bg:
+            from ok.capture.windows.DesktopDuplicationCaptureMethod import DesktopDuplicationCaptureMethod
+            target_method = DesktopDuplicationCaptureMethod
+            capture_method = get_capture(capture_method, target_method, hwnd)
+            return capture_method
+    except Exception as e:
+        logger.error(f'update_capture_method exception, return None: ', e)
+
+
+def get_capture(capture_method, target_method, hwnd):
+    if not isinstance(capture_method, target_method):
+        if capture_method is not None:
+            capture_method.close()
+        capture_method = target_method(hwnd)
+    capture_method.hwnd_window = hwnd
+    return capture_method

{ok-script-0.0.86 → ok-script-0.0.87}/ok/capture/windows/WindowsGraphicsCaptureMethod.py

@@ -23,7 +23,7 @@ logger = get_logger(__name__)
 class WindowsGraphicsCaptureMethod(BaseWindowsCaptureMethod):
     name = "Windows Graphics Capture"
     description = "fast, most compatible, capped at 60fps"
-    last_frame = None
+    last_dx_frame = None
     last_frame_time = 0
     frame_pool = None
     item = None
@@ -38,67 +38,71 @@ class WindowsGraphicsCaptureMethod(BaseWindowsCaptureMethod):
 
     def frame_arrived_callback(self, x, y):
         try:
-            frame = self.frame_pool.TryGetNextFrame()
+            self.last_frame_time = time.time()
+            self.last_dx_frame = self.frame_pool.TryGetNextFrame()
         except Exception as e:
             logger.error(f"TryGetNextFrame error {e}")
             self.close()
             return
+
+    def convert_dx_frame(self):
+        frame = self.last_dx_frame
         if not frame:
+            return None
+
+        need_reset_framepool = False
+        if frame.ContentSize.Width != self.last_size.Width or frame.ContentSize.Height != self.last_size.Height:
+            need_reset_framepool = True
+            self.last_size = frame.ContentSize
+
+        if need_reset_framepool:
+            logger.info('need_reset_framepool')
+            self.reset_framepool(frame.ContentSize)
             return
-        self.last_frame_time = time.time()
-        img = None
-        with frame:
-            need_reset_framepool = False
-            need_reset_device = False
-            if frame.ContentSize.Width != self.last_size.Width or frame.ContentSize.Height != self.last_size.Height:
-                need_reset_framepool = True
-                self.last_size = frame.ContentSize
+        need_reset_device = False
 
-            if need_reset_framepool:
-                self.reset_framepool(frame.ContentSize)
-                return
-            tex = None
+        tex = None
 
-            cputex = None
-            try:
-                from ok.rotypes.Windows.Graphics.DirectX.Direct3D11 import IDirect3DDxgiInterfaceAccess
-                from ok.rotypes.roapi import GetActivationFactory
-                tex = frame.Surface.astype(IDirect3DDxgiInterfaceAccess).GetInterface(
-                    d3d11.ID3D11Texture2D.GUID).astype(d3d11.ID3D11Texture2D)
-                desc = tex.GetDesc()
-                desc2 = d3d11.D3D11_TEXTURE2D_DESC()
-                desc2.Width = desc.Width
-                desc2.Height = desc.Height
-                desc2.MipLevels = desc.MipLevels
-                desc2.ArraySize = desc.ArraySize
-                desc2.Format = desc.Format
-                desc2.SampleDesc = desc.SampleDesc
-                desc2.Usage = d3d11.D3D11_USAGE_STAGING
-                desc2.CPUAccessFlags = d3d11.D3D11_CPU_ACCESS_READ
-                desc2.BindFlags = 0
-                desc2.MiscFlags = 0
-                cputex = self.dxdevice.CreateTexture2D(ctypes.byref(desc2), None)
-                self.immediatedc.CopyResource(cputex, tex)
-                mapinfo = self.immediatedc.Map(cputex, 0, d3d11.D3D11_MAP_READ, 0)
-                img = np.ctypeslib.as_array(ctypes.cast(mapinfo.pData, PBYTE),
-                                            (desc.Height, mapinfo.RowPitch // 4, 4))[
-                      :, :desc.Width].copy()
-                self.immediatedc.Unmap(cputex, 0)
-            except OSError as e:
-                if e.winerror == d3d11.DXGI_ERROR_DEVICE_REMOVED or e.winerror == d3d11.DXGI_ERROR_DEVICE_RESET:
-                    need_reset_framepool = True
-                    need_reset_device = True
-                else:
-                    raise
-            finally:
-                if tex is not None:
-                    tex.Release()
-                if cputex is not None:
-                    cputex.Release()
-            if need_reset_framepool:
-                self.reset_framepool(frame.ContentSize, need_reset_device)
-                return self.get_frame()
-        self.last_frame = img
+        cputex = None
+        try:
+            from ok.rotypes.Windows.Graphics.DirectX.Direct3D11 import IDirect3DDxgiInterfaceAccess
+            from ok.rotypes.roapi import GetActivationFactory
+            tex = frame.Surface.astype(IDirect3DDxgiInterfaceAccess).GetInterface(
+                d3d11.ID3D11Texture2D.GUID).astype(d3d11.ID3D11Texture2D)
+            desc = tex.GetDesc()
+            desc2 = d3d11.D3D11_TEXTURE2D_DESC()
+            desc2.Width = desc.Width
+            desc2.Height = desc.Height
+            desc2.MipLevels = desc.MipLevels
+            desc2.ArraySize = desc.ArraySize
+            desc2.Format = desc.Format
+            desc2.SampleDesc = desc.SampleDesc
+            desc2.Usage = d3d11.D3D11_USAGE_STAGING
+            desc2.CPUAccessFlags = d3d11.D3D11_CPU_ACCESS_READ
+            desc2.BindFlags = 0
+            desc2.MiscFlags = 0
+            cputex = self.dxdevice.CreateTexture2D(ctypes.byref(desc2), None)
+            self.immediatedc.CopyResource(cputex, tex)
+            mapinfo = self.immediatedc.Map(cputex, 0, d3d11.D3D11_MAP_READ, 0)
+            img = np.ctypeslib.as_array(ctypes.cast(mapinfo.pData, PBYTE),
+                                        (desc.Height, mapinfo.RowPitch // 4, 4))[
+                  :, :desc.Width].copy()
+            self.immediatedc.Unmap(cputex, 0)
+            return img
+        except OSError as e:
+            if e.winerror == d3d11.DXGI_ERROR_DEVICE_REMOVED or e.winerror == d3d11.DXGI_ERROR_DEVICE_RESET:
+                need_reset_framepool = True
+                need_reset_device = True
+            else:
+                raise e
+        finally:
+            if tex is not None:
+                tex.Release()
+            if cputex is not None:
+                cputex.Release()
+        if need_reset_framepool:
+            self.reset_framepool(frame.ContentSize, need_reset_device)
+            return self.get_frame()
 
     @property
     def hwnd_window(self):
@@ -139,7 +143,7 @@ class WindowsGraphicsCaptureMethod(BaseWindowsCaptureMethod):
                 self.evtoken = item.add_Closed(delegate)
                 self.frame_pool = Direct3D11CaptureFramePool.CreateFreeThreaded(self.rtdevice,
                                                                                 DirectXPixelFormat.B8G8R8A8UIntNormalized,
-                                                                                1, item.Size)
+                                                                                2, item.Size)
                 self.session = self.frame_pool.CreateCaptureSession(item)
                 pool = self.frame_pool
                 pool.add_FrameArrived(
@@ -193,10 +197,11 @@ class WindowsGraphicsCaptureMethod(BaseWindowsCaptureMethod):
     @override
     def do_get_frame(self):
         if self.start_or_stop():
-            frame = self.last_frame
-            self.last_frame = None
+            frame = self.convert_dx_frame()
+            if frame is None:
+                return
+            self.last_dx_frame = None
             latency = time.time() - self.last_frame_time
-            self.last_frame_time = time.time()
 
             frame = self.crop_image(frame)
 
@@ -209,14 +214,16 @@ class WindowsGraphicsCaptureMethod(BaseWindowsCaptureMethod):
                 logger.warning(f"latency too large return None frame: {latency}")
                 return None
             else:
+                # logger.debug(f'frame latency: {latency}')
                 return frame
 
     def reset_framepool(self, size, reset_device=False):
+        logger.info(f'reset_framepool')
         from ok.rotypes.Windows.Graphics.DirectX import DirectXPixelFormat
         if reset_device:
             self.create_device()
         self.frame_pool.Recreate(self.rtdevice,
-                                 DirectXPixelFormat.B8G8R8A8UIntNormalized, 1, size)
+                                 DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, size)
 
     def crop_image(self, frame):
         if frame is not None:

ok-script-0.0.87/ok/feature/CompressCoco.py

@@ -0,0 +1,57 @@
+import json
+import os
+import sys
+
+import cv2
+from PIL import Image
+
+
+def compress_coco(coco_json) -> None:
+    coco_folder = os.path.dirname(coco_json)
+
+    image_dict = {}
+    with open(coco_json, 'r') as file:
+        data = json.load(file)
+
+        image_map = {image['id']: image['file_name'] for image in data['images']}
+        category_map = {category['id']: category['name'] for category in data['categories']}
+
+        for annotation in data['annotations']:
+            image_id = annotation['image_id']
+            category_id = annotation['category_id']
+            bbox = annotation['bbox']
+
+            # Load and scale the image
+            image_path = str(os.path.join(coco_folder, image_map[image_id]))
+            image = cv2.imread(image_path)
+            original_height, original_width = image.shape[:2]
+            if image is None:
+                print(f'Could not read image {image_path}')
+                continue
+            x, y, w, h = bbox
+            # Crop the image to the bounding box
+            image = image[round(y):round(y + h), round(x):round(x + w), :3]
+
+            image_features = image_dict.get(image_path, [])
+            image_features.append((round(x), round(y), image, original_width, original_height))
+            image_dict[image_path] = image_features
+
+        # Loop through the image_dict and write all the image_feature associated with it in a new PNG
+        for image_path, features in image_dict.items():
+            background = None
+            for feature in features:
+                x, y, image, original_width, original_height = feature
+                # Create a white background
+                if not background:
+                    background = Image.new('RGBA', (original_width, original_height), (255, 255, 255, 255))
+                # Convert the OpenCV image (numpy array) to a PIL image
+                image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGBA))
+                # Paste the image onto the background
+                background.paste(image, (x, y), image)
+                # Save the image with compression level 9
+            background.save(image_path, 'PNG', optimize=True, compress_level=9)
+
+
+if __name__ == '__main__':
+    json_file = sys.argv[1]
+    compress_coco(json_file)

{ok-script-0.0.86 → ok-script-0.0.87}/ok/feature/FeatureSet.py

@@ -1,6 +1,7 @@
 import json
 import os
 import sys
+import threading
 from typing import Dict
 from typing import List
 
@@ -21,7 +22,7 @@ class FeatureSet:
     feature_dict: Dict[str, Feature] = {}
     box_dict: Dict[str, Box] = {}
 
-    def __init__(self, coco_json: str, default_horizontal_variance=0,
+    def __init__(self, debug, coco_json: str, default_horizontal_variance=0,
                  default_vertical_variance=0, default_threshold=0.95) -> None:
         """
         Initialize the FeatureSet by loading images and annotations from a COCO dataset.
@@ -33,6 +34,7 @@ class FeatureSet:
         """
         self.coco_json = resource_path(coco_json)
         self.coco_folder = os.path.dirname(self.coco_json)
+        self.debug = debug
 
         logger.debug(f'Loading features from {self.coco_json}')
 
@@ -44,6 +46,7 @@ class FeatureSet:
         self.default_threshold = default_threshold
         self.default_horizontal_variance = default_horizontal_variance
         self.default_vertical_variance = default_vertical_variance
+        self.lock = threading.Lock()
 
     def check_size(self, frame):
         height, width = frame.shape[:2]
@@ -52,6 +55,8 @@ class FeatureSet:
             self.width = width
             self.height = height
             self.process_data()
+        elif not self.feature_dict:
+            self.process_data()
 
     def process_data(self) -> None:
         """
@@ -61,49 +66,60 @@ class FeatureSet:
             width (int): Target width for scaling images.
             height (int): Target height for scaling images.
         """
-        self.feature_dict.clear()
-        self.box_dict.clear()
-        with open(self.coco_json, 'r') as file:
-            data = json.load(file)
-
-        # Create a map from image ID to file name
-        image_map = {image['id']: image['file_name'] for image in data['images']}
-
-        # Create a map from category ID to category name
-        category_map = {category['id']: category['name'] for category in data['categories']}
-
-        for annotation in data['annotations']:
-            image_id = annotation['image_id']
-            category_id = annotation['category_id']
-            bbox = annotation['bbox']
-
-            # Load and scale the image
-            image_path = str(os.path.join(self.coco_folder, image_map[image_id]))
-            image = cv2.imread(image_path)
-            _, original_width = image.shape[:2]
-            if image is None:
-                logger.error(f'Could not read image {image_path}')
-                continue
-            x, y, w, h = bbox
-            if self.width != image.shape[1] or self.height != image.shape[0]:
-                scale_x, scale_y = self.width / image.shape[1], self.height / image.shape[0]
-                logger.debug(f'scaling images {scale_x}, {scale_y}')
-                image = cv2.resize(image, (self.width, self.height))
-                # Calculate the scaled bounding box
-                x, y, w, h = x * scale_x, y * scale_y, w * scale_x, h * scale_y
-
-            # Crop the image to the bounding box
-            image = image[int(y):int(y + h), int(x):int(x + w), :3]
-
-            # Store in featureDict using the category name
-            category_name = category_map[category_id]
-            logger.debug(
-                f"loaded {category_name} resized width {self.width} / original_width:{original_width},scale_x:{self.width / original_width}")
-            if category_name in self.feature_dict:
-                raise ValueError(f"Multiple boxes found for category {category_name}")
-            if not category_name.startswith('box_'):
-                self.feature_dict[category_name] = Feature(image, x, y, w, h)
-            self.box_dict[category_name] = Box(x, y, w, h, name=category_name)
+        with self.lock:
+            if self.debug:
+                folder_size = sum(os.path.getsize(os.path.join(dirpath, filename))
+                                  for dirpath, dirnames, filenames in os.walk(self.coco_folder)
+                                  for filename in filenames)
+                # Convert size to MB
+                folder_size_mb = folder_size / (1024 * 1024)
+                if folder_size_mb > 5:
+                    from ok.feature.CompressCoco import compress_coco
+                    logger.info(f'template folder greater than 5MB try to compress the COCO dataset')
+                    compress_coco(self.coco_json)
+            self.feature_dict.clear()
+            self.box_dict.clear()
+            with open(self.coco_json, 'r') as file:
+                data = json.load(file)
+
+            # Create a map from image ID to file name
+            image_map = {image['id']: image['file_name'] for image in data['images']}
+
+            # Create a map from category ID to category name
+            category_map = {category['id']: category['name'] for category in data['categories']}
+
+            for annotation in data['annotations']:
+                image_id = annotation['image_id']
+                category_id = annotation['category_id']
+                bbox = annotation['bbox']
+
+                # Load and scale the image
+                image_path = str(os.path.join(self.coco_folder, image_map[image_id]))
+                image = cv2.imread(image_path)
+                _, original_width = image.shape[:2]
+                if image is None:
+                    logger.error(f'Could not read image {image_path}')
+                    continue
+                x, y, w, h = bbox
+                if self.width != image.shape[1] or self.height != image.shape[0]:
+                    scale_x, scale_y = self.width / image.shape[1], self.height / image.shape[0]
+                    logger.debug(f'scaling images {scale_x}, {scale_y}')
+                    image = cv2.resize(image, (self.width, self.height))
+                    # Calculate the scaled bounding box
+                    x, y, w, h = x * scale_x, y * scale_y, w * scale_x, h * scale_y
+
+                # Crop the image to the bounding box
+                image = image[round(y):round(y + h), round(x):round(x + w), :3]
+
+                # Store in featureDict using the category name
+                category_name = category_map[category_id]
+                logger.debug(
+                    f"loaded {category_name} resized width {self.width} / original_width:{original_width},scale_x:{self.width / original_width}")
+                if category_name in self.feature_dict:
+                    raise ValueError(f"Multiple boxes found for category {category_name}")
+                if not category_name.startswith('box_'):
+                    self.feature_dict[category_name] = Feature(image, x, y, w, h)
+                self.box_dict[category_name] = Box(x, y, w, h, name=category_name)
 
     def get_box_by_name(self, mat, category_name: str) -> Box:
         self.check_size(mat)
@@ -130,7 +146,6 @@ class FeatureSet:
 
             # Save the image
             cv2.imwrite(file_path, image.mat)
-            print(f"Saved {file_path}")
 
     def find_one(self, mat: np.ndarray, category_name: str, horizontal_variance: float = 0,
                  vertical_variance: float = 0,
@@ -183,16 +198,16 @@ class FeatureSet:
                 width = to_x - x
             if height == -1:
                 height = to_y - y
-            search_x1 = int(x * frame_width)
-            search_y1 = int(y * frame_height)
-            search_x2 = int((x + width) * frame_width)
-            search_y2 = int((y + height) * frame_height)
+            search_x1 = round(x * frame_width)
+            search_y1 = round(y * frame_height)
+            search_x2 = round((x + width) * frame_width)
+            search_y2 = round((y + height) * frame_height)
         else:
             # Define search area using variance
-            search_x1 = max(0, int(feature.x - self.width * horizontal_variance))
-            search_y1 = max(0, int(feature.y - self.height * vertical_variance))
-            search_x2 = min(self.width, int(feature.x + feature_width + self.width * horizontal_variance))
-            search_y2 = min(self.height, int(feature.y + feature_height + self.height * vertical_variance))
+            search_x1 = max(0, round(feature.x - self.width * horizontal_variance))
+            search_y1 = max(0, round(feature.y - self.height * vertical_variance))
+            search_x2 = min(self.width, round(feature.x + feature_width + self.width * horizontal_variance))
+            search_y2 = min(self.height, round(feature.y + feature_height + self.height * vertical_variance))
 
         search_area = mat[search_y1:search_y2, search_x1:search_x2, :3]
         # Crop the search area from the image

{ok-script-0.0.86 → ok-script-0.0.87}/ok/gui/debug/LogWindow.py

@@ -60,28 +60,25 @@ class LogModel(QAbstractListModel):
         # Create colored text based on level
         color_format = self.get_color_format(level)
         colored_text = ColoredText(message, color_format, level)
-
+        self.logs.append(colored_text)
         if len(self.logs) >= 500:
             self.beginRemoveRows(QModelIndex(), 0, 0)
             self.logs.pop(0)
             self.endRemoveRows()
 
         self.beginInsertRows(QModelIndex(), self.rowCount(QModelIndex()), self.rowCount(QModelIndex()))
-        self.logs.append(colored_text)
+        self.do_filter_logs()
         self.endInsertRows()
 
-        self.filter_logs(self.current_level, self.current_keyword)
+    def do_filter_logs(self):
 
-    def filter_logs(self, level, keyword):
-        self.current_level = level
-        self.current_keyword = keyword
-        keyword = keyword.lower()
+        keyword = self.current_keyword.lower()
 
         # Get the numeric severity for the current level
-        current_level_severity = level_severity.get(level, 0)
+        current_level_severity = level_severity.get(self.current_level, 0)
 
         # Filter logs based on severity level and keyword
-        if level == "ALL":
+        if self.current_level == "ALL":
             self.filtered_logs = [log for log in self.logs if keyword in log.text.lower()]
         else:
             self.filtered_logs = [
@@ -90,6 +87,10 @@ class LogModel(QAbstractListModel):
                    and keyword in log.text.lower()
             ]
 
+    def filter_logs(self, level, keyword):
+        self.current_level = level
+        self.current_keyword = keyword
+        self.do_filter_logs()
         self.layoutChanged.emit()
         self.log_list.scrollToBottom()
 

{ok-script-0.0.86 → ok-script-0.0.87}/ok/gui/tasks/TaskOpButton.py

@@ -15,19 +15,13 @@ class TaskOpButton(QPushButton):
 
     def update_task(self, task: BaseTask):
         if task.enabled:
-            if task.done:
-                self.setText(self.tr("Rerun"))
-            else:
-                self.setText(self.tr("Disable"))
+            self.setText(self.tr("Disable"))
         else:
             self.setText(self.tr("Enable"))
 
     @Slot()
     def toggle_text(self):
         if self.task.enabled:
-            if self.task.done:
-                self.task.enable()
-            else:
-                self.task.disable()
+            self.task.disable()
         else:
             self.task.disable()

{ok-script-0.0.86 → ok-script-0.0.87}/ok/task/BaseTask.py

@@ -1,4 +1,5 @@
 import threading
+import time
 
 from ok.config.Config import Config
 from ok.config.InfoDict import InfoDict
@@ -27,8 +28,17 @@ class BaseTask(ExecutorOperation):
         self._paused = False
         self.lock = threading.Lock()
         self._handler = None
-        self._done = False
         self.running = False
+        self.check_trigger_interval = 0
+        self.last_check_trigger_time = 0
+
+    def should_check_trigger(self):
+        now = time.time()
+        time_diff = now - self.last_check_trigger_time
+        if time_diff > self.check_trigger_interval:
+            self.last_check_trigger_time = now
+            return True
+        return False
 
     def get_status(self):
         if self.running:
@@ -46,15 +56,6 @@ class BaseTask(ExecutorOperation):
             self._enabled = True
             self.info_clear()
         communicate.task.emit(self)
-        self._done = False
-
-    @property
-    def done(self):
-        return self._done
-
-    def set_done(self):
-        self._done = True
-        self.disable()
 
     @property
     def handler(self) -> Handler:
@@ -156,10 +157,7 @@ class BaseTask(ExecutorOperation):
     def run(self):
         pass
 
-    def check_trigger(self):
-        return True
-
-    def check_condition(self):
+    def trigger(self):
         return True
 
     def on_destroy(self):

{ok-script-0.0.86 → ok-script-0.0.87}/ok/task/ExecutorOperation.py

@@ -34,9 +34,11 @@ class ExecutorOperation:
     def is_scene(self, the_scene):
         return isinstance(self.executor.current_scene, the_scene)
 
-    def click(self, x=-1, y=-1, move_back=False, name=None):
-        frame = self.executor.nullable_frame()
-        communicate.emit_draw_box("click", [Box(max(0, x - 10), max(0, y - 10), 20, 20, name="click")], "green", frame)
+    def click(self, x: int | Box = -1, y=-1, move_back=False, name=None):
+        if isinstance(x, Box):
+            self.click_box(x, move_back=move_back)
+            return
+        communicate.emit_draw_box("click", [Box(max(0, x - 10), max(0, y - 10), 20, 20, name="click")], "green")
         self.executor.reset_scene()
         self.executor.interaction.click(x, y, move_back, name=name)
 
@@ -132,7 +134,8 @@ class ExecutorOperation:
     def move_relative(self, x, y):
         self.executor.interaction.move_relative(x, y)
 
-    def click_box(self, box: Box | List[Box] = None, relative_x=0.5, relative_y=0.5, raise_if_not_found=True):
+    def click_box(self, box: Box | List[Box] = None, relative_x=0.5, relative_y=0.5, raise_if_not_found=True,
+                  move_back=False):
         if isinstance(box, list):
             if len(box) > 0:
                 box = box[0]
@@ -143,7 +146,7 @@ class ExecutorOperation:
                 raise Exception(f"click_box box is None")
             return
         x, y = box.relative_with_variance(relative_x, relative_y)
-        self.click(x, y, name=box.name)
+        self.click(x, y, name=box.name, move_back=move_back)
 
     def wait_scene(self, scene_type=None, time_out=0, pre_action=None, post_action=None):
         return self.executor.wait_scene(scene_type, time_out, pre_action, post_action)