visus-desktop 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visus/desktop/__init__.py +3468 -0
- visus/desktop/cli/__init__.py +103 -0
- visus/desktop/cli/_common.py +190 -0
- visus/desktop/cli/_output.py +56 -0
- visus/desktop/cli/commands/__init__.py +1 -0
- visus/desktop/cli/commands/capture.py +70 -0
- visus/desktop/cli/commands/clipboard.py +32 -0
- visus/desktop/cli/commands/config.py +143 -0
- visus/desktop/cli/commands/doctor.py +30 -0
- visus/desktop/cli/commands/image.py +218 -0
- visus/desktop/cli/commands/keyboard.py +52 -0
- visus/desktop/cli/commands/mouse.py +118 -0
- visus/desktop/cli/commands/native.py +190 -0
- visus/desktop/cli/commands/process.py +93 -0
- visus/desktop/cli/commands/project.py +86 -0
- visus/desktop/cli/commands/screen.py +93 -0
- visus/desktop/cli/commands/smart.py +86 -0
- visus/desktop/cli/commands/text.py +292 -0
- visus/desktop/cli/commands/window.py +146 -0
- visus/desktop/cli/repl.py +44 -0
- visus/desktop/core/__init__.py +52 -0
- visus/desktop/core/capabilities.py +538 -0
- visus/desktop/core/elements.py +218 -0
- visus/desktop/core/image_map.py +102 -0
- visus/desktop/core/image_processing.py +276 -0
- visus/desktop/core/keyboard_commands.py +424 -0
- visus/desktop/core/keyboard_extra.py +517 -0
- visus/desktop/core/locate.py +328 -0
- visus/desktop/core/locators.py +324 -0
- visus/desktop/core/mouse_extra.py +117 -0
- visus/desktop/core/native.py +252 -0
- visus/desktop/core/native_rpa.py +482 -0
- visus/desktop/core/ocr_engine.py +974 -0
- visus/desktop/core/overlay.py +600 -0
- visus/desktop/core/overlay_win32.py +403 -0
- visus/desktop/core/process.py +151 -0
- visus/desktop/core/relative_image.py +232 -0
- visus/desktop/core/report.py +607 -0
- visus/desktop/core/screen.py +369 -0
- visus/desktop/core/session.py +275 -0
- visus/desktop/core/smart.py +198 -0
- visus/desktop/core/task_executor.py +1802 -0
- visus/desktop/core/virtual_mouse.py +548 -0
- visus/desktop/core/wait_manager.py +317 -0
- visus/desktop/core/window.py +430 -0
- visus/desktop/core/window_linux.py +83 -0
- visus/desktop/doctor.py +198 -0
- visus/desktop/exceptions.py +88 -0
- visus/desktop/mcp_server.py +2111 -0
- visus/desktop/platform.py +51 -0
- visus/desktop/py.typed +0 -0
- visus/desktop/resources/cli.md +503 -0
- visus/desktop/resources/mcp.md +485 -0
- visus/desktop/utils/__init__.py +32 -0
- visus/desktop/utils/clipboard.py +68 -0
- visus/desktop/utils/config.py +585 -0
- visus/desktop/utils/files.py +89 -0
- visus/desktop/utils/retry.py +96 -0
- visus/desktop/utils/text_filters.py +170 -0
- visus_desktop-0.1.0.dist-info/METADATA +1106 -0
- visus_desktop-0.1.0.dist-info/RECORD +66 -0
- visus_desktop-0.1.0.dist-info/WHEEL +5 -0
- visus_desktop-0.1.0.dist-info/entry_points.txt +7 -0
- visus_desktop-0.1.0.dist-info/licenses/LICENSE +21 -0
- visus_desktop-0.1.0.dist-info/top_level.txt +2 -0
- visus_pytest/__init__.py +202 -0
|
@@ -0,0 +1,3468 @@
|
|
|
1
|
+
r"""
|
|
2
|
+
Visus Desktop — Advanced GUI Automation (OCR + image + native control).
|
|
3
|
+
|
|
4
|
+
A Python library to automate Windows desktop apps by four interchangeable
|
|
5
|
+
techniques: native accessibility control (no pixels), template-image matching,
|
|
6
|
+
OCR text matching, and absolute coordinates. Pick a technique, or let the
|
|
7
|
+
smart cascade try them in order. Everything is also exposed as an MCP server
|
|
8
|
+
(for AI agents) and a terminal CLI (`visus`).
|
|
9
|
+
|
|
10
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
11
|
+
⚠️ REQUIREMENT: TESSERACT OCR (only for the OCR features)
|
|
12
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
13
|
+
1. Download: https://github.com/UB-Mannheim/tesseract/wiki
|
|
14
|
+
2. Install to: C:\\Program Files\\Tesseract-OCR\\
|
|
15
|
+
3. Add it to the system PATH
|
|
16
|
+
4. Test: tesseract --version
|
|
17
|
+
(Not needed if you set config['ocr_backend']='rapidocr' or only use
|
|
18
|
+
native / image / coordinate targeting. Run `visus doctor` to check.)
|
|
19
|
+
|
|
20
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
21
|
+
📦 INSTALLATION
|
|
22
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
23
|
+
pip install visus-desktop # core
|
|
24
|
+
pip install "visus-desktop[native]" # + pywinauto (native RPA, Win)
|
|
25
|
+
pip install "visus-desktop[mcp]" # + MCP server (visus-mcp)
|
|
26
|
+
pip install "visus-desktop[rapidocr]" # + ONNX OCR backend (no Tesseract)
|
|
27
|
+
|
|
28
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
29
|
+
🚀 QUICKSTART
|
|
30
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
31
|
+
>>> from visus.desktop import VisusBot, ensure_window
|
|
32
|
+
>>> ensure_window(process="notepad.exe") # open/restore/focus first
|
|
33
|
+
>>> bot = VisusBot()
|
|
34
|
+
>>> bot.click_text("Login") # OCR-find text and click
|
|
35
|
+
>>> bot.type_text("admin") # type into focused field
|
|
36
|
+
>>> bot.click_image("save_button.png") # template-match and click
|
|
37
|
+
|
|
38
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
39
|
+
🧱 THE API — every public name, by category
|
|
40
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
41
|
+
|
|
42
|
+
▶ MAIN CLASS
|
|
43
|
+
VisusBot(config=None)
|
|
44
|
+
The central automation object. Key methods:
|
|
45
|
+
.find_text(text, region=None, confidence_threshold=75, occurrence=1)
|
|
46
|
+
.click_text(text, region=None, mouse_button="left", sendtext=None,
|
|
47
|
+
occurrence=1, backtrack=False)
|
|
48
|
+
.find_image(image_path, region=None, confidence=0.9)
|
|
49
|
+
.click_image(image_path, region=None, confidence=0.9,
|
|
50
|
+
mouse_button="left", backtrack=False)
|
|
51
|
+
.click_at((x, y, w, h), mouse_button="left")
|
|
52
|
+
.type_text(text, interval=0.05)
|
|
53
|
+
.keyboard_command(command) # "Enter", "Ctrl+S", ... (case-sensitive)
|
|
54
|
+
.extract_text_from_region(region, filter_type="both", ...)
|
|
55
|
+
.execute_tasks(tasks, resume=False)
|
|
56
|
+
.start_task_session() / .end_task_session() # manual backtrack scope
|
|
57
|
+
BotVision — backward-compatibility alias for VisusBot.
|
|
58
|
+
|
|
59
|
+
▶ TASK-LIST EXECUTION (run a whole flow, with retry/backtrack)
|
|
60
|
+
execute_tasks(tasks, resume=False)
|
|
61
|
+
tasks is a list of dicts. Shapes:
|
|
62
|
+
{"image": "btn.png", "region": (x,y,w,h), "confidence": 0.9,
|
|
63
|
+
"mouse_button": "left", "backtrack": True}
|
|
64
|
+
{"text": "Login", "region": (x,y,w,h), "char_type": "letters",
|
|
65
|
+
"sendtext": "user", "occurrence": 1}
|
|
66
|
+
{"type": "type_text", "text": "hello", "interval": 0.05}
|
|
67
|
+
{"type": "keyboard_command", "command": "Escape"} # case-sensitive token
|
|
68
|
+
{"type": "click", "x": 500, "y": 300, "mouse_button": "right"}
|
|
69
|
+
{"type": "relative_image", "anchor_image": "icon.png",
|
|
70
|
+
"target_image": "ok.png", "max_distance": 200}
|
|
71
|
+
{"type": "extract_text", "region": (x,y,w,h)}
|
|
72
|
+
Any task: backtrack, wait_until_found, wait_until_disappears, delay.
|
|
73
|
+
click_images(...) — legacy task-list helper (compatibility).
|
|
74
|
+
run_automation(...) — legacy `__main__` entry helper (compatibility).
|
|
75
|
+
|
|
76
|
+
▶ FIND / CLICK CONVENIENCE (one-shot, builds a temp bot internally)
|
|
77
|
+
find_text(text, region=None, confidence_threshold=75, occurrence=1) -> box|None
|
|
78
|
+
click_text(text, region=None, mouse_button="left", sendtext=None, ...) -> bool
|
|
79
|
+
find_image(image_path, region=None, confidence=0.9) -> box|None
|
|
80
|
+
click_image(image_path, region=None, confidence=0.9, ...) -> bool
|
|
81
|
+
click_at(x, y, mouse_button="left") -> bool # absolute coords (last resort)
|
|
82
|
+
type_text(text) -> bool
|
|
83
|
+
|
|
84
|
+
▶ RELATIVE-IMAGE (find a target near a unique anchor)
|
|
85
|
+
find_relative_image(anchor_image, target_image, max_distance=200, ...) -> box|None
|
|
86
|
+
click_relative_image(anchor_image, target_image, ...) -> bool
|
|
87
|
+
|
|
88
|
+
▶ COORDINATES / KEYBOARD
|
|
89
|
+
click_coordinates(x, y, mouse_button="left") -> bool
|
|
90
|
+
type_text_standalone(text) -> bool
|
|
91
|
+
keyboard_command_standalone(command) -> bool # e.g. "Ctrl+S", "Escape"
|
|
92
|
+
get_available_keyboard_commands() -> list[str] # the valid, case-sensitive tokens
|
|
93
|
+
|
|
94
|
+
▶ TEXT EXTRACTION (OCR a region, no click)
|
|
95
|
+
extract_text_from_region_standalone(region, filter_type="both",
|
|
96
|
+
return_full_data=False, confidence_threshold=50) -> list|dict
|
|
97
|
+
|
|
98
|
+
▶ SMART CASCADE & COORDINATE-FREE CAPTURE (recommended for robustness)
|
|
99
|
+
smart_find(query=None, image=None, coordinates=None, window=None,
|
|
100
|
+
control_type=None, order=None, ...) -> {found, box, method, tried}
|
|
101
|
+
smart_click(...) -> {clicked, method, tried}
|
|
102
|
+
One call tries native → image → OCR → coordinates (configurable order)
|
|
103
|
+
and reports which won. The simplest robust "find/click this".
|
|
104
|
+
capture_element(label, auto=None, text=None, native=None, region=None,
|
|
105
|
+
window=None, click=False, elements_dir="./visus-elements")
|
|
106
|
+
-> {found, located_by, box, template_path, annotated_path, clicked}
|
|
107
|
+
Locate an element (Win32 menu → OCR → UIA), save a reusable PNG
|
|
108
|
+
template + an annotated screenshot, optionally click it. The
|
|
109
|
+
coordinate-free way to build a click-by-image flow step by step.
|
|
110
|
+
|
|
111
|
+
▶ NATIVE RPA (no pixels — accessibility tree; needs the [native] extra)
|
|
112
|
+
Import from visus.desktop.core.native_rpa:
|
|
113
|
+
native_dump_tree(window=None, pid=None, max_depth=3) -> [ {depth,
|
|
114
|
+
control_type, title, auto_id, rect} ] # "see" the UI structure
|
|
115
|
+
native_click(window/pid, title=, control_type=, auto_id=, method="auto")
|
|
116
|
+
native_set_text(text, ...selector...) ; native_get_text(...) -> str|None
|
|
117
|
+
native_get_state(...) -> {exists,visible,enabled,checked,value,rect}
|
|
118
|
+
native_wait(..., state="ready", timeout=10)
|
|
119
|
+
Immune to theme/DPI/resolution/language; empty on non-accessible UIs
|
|
120
|
+
(Java Swing without Access Bridge) — fall back to OCR/image there.
|
|
121
|
+
|
|
122
|
+
▶ WINDOWS / PROCESSES (fast win32; target the right app first)
|
|
123
|
+
ensure_window(title=None, process=None, show=True, foreground=True)
|
|
124
|
+
-> {found, running, hwnd, pid, title, minimized, maximized,
|
|
125
|
+
foreground, rect, shown, matches}
|
|
126
|
+
★ CALL THIS FIRST. Is the app open / minimized / where? — and brings
|
|
127
|
+
it to the front. Prefer process= (avoids title false-positives).
|
|
128
|
+
list_windows(title=None, process=None) -> [ {hwnd,title,pid,minimized,...} ]
|
|
129
|
+
get_window(title_substring) -> {title,pid,hwnd,x,y,width,height}|None
|
|
130
|
+
focus_window / maximize_window / minimize_window / close_window(title) -> bool
|
|
131
|
+
launch(command, args=None) -> pid ; find_process(name) -> [Process]
|
|
132
|
+
is_process_running(name) -> bool ; kill_process(pid) -> bool
|
|
133
|
+
|
|
134
|
+
▶ SCREEN / MONITORS
|
|
135
|
+
take_screenshot(region=None, monitor=None) # PIL.Image
|
|
136
|
+
save_screenshot(path, region=None, monitor=None) -> {path,width,height}
|
|
137
|
+
capture_screen(region=None, monitor=None) ; get_display_size() -> (w,h)
|
|
138
|
+
get_monitors() -> [Monitor] ; Monitor(index,x,y,width,height,is_primary)
|
|
139
|
+
wait_for_stable_screen(region=None, stable_for=0.5, timeout=15) -> bool
|
|
140
|
+
|
|
141
|
+
▶ FILES / CLIPBOARD
|
|
142
|
+
wait_for_file(path, timeout=10) -> bool
|
|
143
|
+
wait_for_file_stable(path, timeout=30, stable_for=1.0) -> bool # downloads
|
|
144
|
+
copy_to_clipboard(text) / paste() / get_clipboard() -> str
|
|
145
|
+
|
|
146
|
+
▶ MOUSE (low-level gestures)
|
|
147
|
+
scroll(clicks) drag(x1,y1,x2,y2) triple_click(x,y)
|
|
148
|
+
move_random(...) mouse_down(button) mouse_up(button)
|
|
149
|
+
|
|
150
|
+
▶ IMAGE MAP / MULTI-MATCH
|
|
151
|
+
ImageMap(mapping) # named templates -> reuse by key
|
|
152
|
+
find_all(image, region=None) ; find_multiple(images)
|
|
153
|
+
|
|
154
|
+
▶ SESSIONS / REPORTS
|
|
155
|
+
Session.start_recording(zip) # context manager around execute_tasks
|
|
156
|
+
Session.replay(zip) -> [events] # reads events; does NOT re-run
|
|
157
|
+
Session.write_report(zip, html) # styled HTML report
|
|
158
|
+
|
|
159
|
+
▶ CORE CLASSES (advanced)
|
|
160
|
+
TaskExecutor, TaskResult, OCREngine, OCRResult, ImageProcessor,
|
|
161
|
+
VisualOverlay, RelativeImageDetector, KeyboardCommander, Backend
|
|
162
|
+
|
|
163
|
+
▶ CONFIG / UTILITIES
|
|
164
|
+
BotVisionConfig(config_dict=None) get_default_config() -> dict
|
|
165
|
+
Hover the class for the full reference grouped by topic. Pass a
|
|
166
|
+
partial dict; everything else uses the defaults. Common keys:
|
|
167
|
+
confidence_threshold, retry_attempts, default_delay,
|
|
168
|
+
show_overlay / overlay_color / overlay_duration / overlay_width,
|
|
169
|
+
default_mouse_button, use_virtual_mouse, click_duration,
|
|
170
|
+
movement_duration, matcher ("cv2"|"pyscreeze"),
|
|
171
|
+
ocr_backend ("tesseract"|"rapidocr"), ocr_languages,
|
|
172
|
+
wait_timeout, screenshot_delay, log_level, log_format
|
|
173
|
+
("text"|"json"), artifacts_dir, screenshot_on_failure.
|
|
174
|
+
``log_level`` and ``log_format`` take effect immediately on
|
|
175
|
+
construction; mutate at runtime via cfg.config[key] = value.
|
|
176
|
+
retry(...) # decorator: re-run a callable on transient failure
|
|
177
|
+
clean_text(s) / limpar_texto(s) matches_filter(text, filter_type)
|
|
178
|
+
preprocess_image_for_ocr(img) find_text_with_multiple_preprocessing(...)
|
|
179
|
+
show_overlay(region) locate_image_with_retry(image, ...)
|
|
180
|
+
|
|
181
|
+
▶ EXCEPTIONS
|
|
182
|
+
BotVisionError (base), ImageNotFoundError, TextNotFoundError,
|
|
183
|
+
TaskExecutionError, ImageProcessingError, OCRProcessingError,
|
|
184
|
+
TesseractNotFoundError, InvalidRegionError, ConfigurationError,
|
|
185
|
+
NotSupportedOnThisOS, NativeError
|
|
186
|
+
|
|
187
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
188
|
+
🤖 USING THE MCP SERVER (drive everything from an AI agent)
|
|
189
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
190
|
+
Install + run:
|
|
191
|
+
pip install "visus-desktop[mcp]"
|
|
192
|
+
visus-mcp # stdio transport (what hosts expect)
|
|
193
|
+
|
|
194
|
+
Wire into Claude Code / Desktop / Cursor (settings.json):
|
|
195
|
+
{ "mcpServers": { "visus-desktop": { "command": "visus-mcp" } } }
|
|
196
|
+
Restart the host; 57 tools become available to the agent.
|
|
197
|
+
|
|
198
|
+
Tool groups (each tool's docstring tells the agent when/how to use it):
|
|
199
|
+
• Targeting: ensure_window ★, list_windows, get_window
|
|
200
|
+
• Find/click: find_text, find_all_text, click_text, find_image,
|
|
201
|
+
click_image, click_at, hybrid_find
|
|
202
|
+
• Smart/capture: smart_find, smart_click, capture_element
|
|
203
|
+
• Native RPA: native_dump_tree, native_click, native_set_text,
|
|
204
|
+
native_get_text, native_get_state, native_wait
|
|
205
|
+
• Keyboard: type_text, keyboard_command
|
|
206
|
+
• Screen: take_screenshot, save_screenshot, get_monitors,
|
|
207
|
+
wait_for_stable_screen
|
|
208
|
+
• Files/clip: wait_for_file, wait_for_file_stable, get_clipboard,
|
|
209
|
+
copy_to_clipboard, extract_text_from_region
|
|
210
|
+
• Process/win: launch, find_process, kill_process, focus_window,
|
|
211
|
+
maximize_window, minimize_window, close_window
|
|
212
|
+
• Task lists: execute_tasks
|
|
213
|
+
• Sessions: start_recording, stop_recording, replay_session, render_report
|
|
214
|
+
• Diagnostics: doctor, configure
|
|
215
|
+
|
|
216
|
+
Recommended agent flow:
|
|
217
|
+
ensure_window(process="app.exe") # open/focus, get rect
|
|
218
|
+
native_dump_tree(window="App") # if accessible -> act natively
|
|
219
|
+
capture_element(label="save", auto="Save", window="App", click=True)
|
|
220
|
+
# or smart_click(query="Save", window="App")
|
|
221
|
+
start_recording("run.zip"); execute_tasks([...]); stop_recording()
|
|
222
|
+
render_report("run.zip", "run.html")
|
|
223
|
+
Full reference: docs/mcp.md. Agent playbooks: skills/.
|
|
224
|
+
|
|
225
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
226
|
+
⌨️ TERMINAL CLI — "Visus CLI Pro" (the `visus` command)
|
|
227
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
228
|
+
Mirrors the MCP surface in the shell (great for scripts/CI). --json on any:
|
|
229
|
+
visus window ensure --process notepad.exe
|
|
230
|
+
visus smart click --query "Login" --dry-run
|
|
231
|
+
visus capture save --auto "Save" --click
|
|
232
|
+
visus native dump-tree --window "Notepad" --json
|
|
233
|
+
visus repl # interactive shell
|
|
234
|
+
visus doctor # environment health check
|
|
235
|
+
Full reference: docs/cli.md.
|
|
236
|
+
|
|
237
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
238
|
+
💡 TIPS & TROUBLESHOOTING
|
|
239
|
+
═══════════════════════════════════════════════════════════════════════════════
|
|
240
|
+
• Always ensure_window(process=) first; scope find/click with region=.
|
|
241
|
+
• backtrack=True for fragile multi-step flows; retry() for arbitrary calls.
|
|
242
|
+
• Element not found → lower confidence / widen region / re-capture template.
|
|
243
|
+
• Inaccurate OCR → narrow region, filter_type="letters", or try rapidocr.
|
|
244
|
+
• Wrong window clicked → match by process, not title; pass the window rect.
|
|
245
|
+
• Run `visus doctor` (or the doctor MCP tool) to diagnose the environment.
|
|
246
|
+
"""
|
|
247
|
+
|
|
248
|
+
import functools
|
|
249
|
+
import logging
|
|
250
|
+
import time
|
|
251
|
+
|
|
252
|
+
from .core.capabilities import CapabilitiesMixin
|
|
253
|
+
from .core.elements import capture_element
|
|
254
|
+
from .core.image_map import ImageMap
|
|
255
|
+
from .core.image_processing import ImageProcessor, preprocess_image_for_ocr
|
|
256
|
+
from .core.keyboard_commands import KeyboardCommander # NEW FEATURE
|
|
257
|
+
from .core.keyboard_extra import KeyboardMixin
|
|
258
|
+
from .core.locate import find_all, find_multiple
|
|
259
|
+
from .core.locators import get_window
|
|
260
|
+
from .core.mouse_extra import (
|
|
261
|
+
drag,
|
|
262
|
+
mouse_down,
|
|
263
|
+
mouse_up,
|
|
264
|
+
move_random,
|
|
265
|
+
scroll,
|
|
266
|
+
triple_click,
|
|
267
|
+
)
|
|
268
|
+
from .core.native import Backend, NativeError
|
|
269
|
+
from .core.ocr_engine import OCREngine, OCRResult, find_text_with_multiple_preprocessing
|
|
270
|
+
from .core.overlay import VisualOverlay, show_overlay
|
|
271
|
+
from .core.process import find_process, is_process_running, kill_process, launch
|
|
272
|
+
from .core.relative_image import RelativeImageDetector # NEW FEATURE
|
|
273
|
+
from .core.screen import (
|
|
274
|
+
Monitor,
|
|
275
|
+
capture_screen,
|
|
276
|
+
get_display_size,
|
|
277
|
+
get_monitors,
|
|
278
|
+
save_screenshot,
|
|
279
|
+
take_screenshot,
|
|
280
|
+
wait_for_stable_screen,
|
|
281
|
+
)
|
|
282
|
+
from .core.session import Session
|
|
283
|
+
from .core.smart import smart_click, smart_find
|
|
284
|
+
|
|
285
|
+
# Import the main classes
|
|
286
|
+
from .core.task_executor import TaskExecutor, TaskResult
|
|
287
|
+
from .core.window import (
|
|
288
|
+
close_window,
|
|
289
|
+
ensure_window,
|
|
290
|
+
focus_window,
|
|
291
|
+
list_windows,
|
|
292
|
+
maximize_window,
|
|
293
|
+
minimize_window,
|
|
294
|
+
)
|
|
295
|
+
from .exceptions import (
|
|
296
|
+
BotVisionError,
|
|
297
|
+
ConfigurationError,
|
|
298
|
+
ImageNotFoundError,
|
|
299
|
+
ImageProcessingError,
|
|
300
|
+
InvalidRegionError,
|
|
301
|
+
NotSupportedOnThisOS,
|
|
302
|
+
OCRProcessingError,
|
|
303
|
+
TaskExecutionError,
|
|
304
|
+
TesseractNotFoundError,
|
|
305
|
+
TextNotFoundError,
|
|
306
|
+
)
|
|
307
|
+
from .utils.clipboard import copy as copy_to_clipboard
|
|
308
|
+
from .utils.clipboard import get_clipboard, paste
|
|
309
|
+
from .utils.config import BotVisionConfig, get_default_config
|
|
310
|
+
from .utils.files import wait_for_file, wait_for_file_stable
|
|
311
|
+
from .utils.retry import retry
|
|
312
|
+
from .utils.text_filters import clean_text, limpar_texto, matches_filter
|
|
313
|
+
|
|
314
|
+
# Library version
|
|
315
|
+
__version__ = "0.1.0"
|
|
316
|
+
__author__ = "Matheus Pereira"
|
|
317
|
+
__email__ = "matheuszwilkdev@gmail.com"
|
|
318
|
+
__license__ = "MIT"
|
|
319
|
+
|
|
320
|
+
# Default logging configuration
|
|
321
|
+
logger = logging.getLogger(__name__)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
# Keys we copy from the wrapped method's kwargs into the recorded event
|
|
325
|
+
# so reports show the relevant context per call (the executor's own
|
|
326
|
+
# ``_emit_task_event`` picks these up via ``_task_type`` / ``_task_target``).
|
|
327
|
+
_RECORDED_KWARGS = (
|
|
328
|
+
"text", "image", "image_path", "anchor_image", "target_image",
|
|
329
|
+
"command", "region", "x", "y", "confidence", "confidence_threshold",
|
|
330
|
+
"mouse_button", "occurrence", "specific", "max_distance", "filter_type",
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _records_in_session(type_field):
|
|
335
|
+
"""Decorator: emit a structured ``task_event`` for the wrapped call.
|
|
336
|
+
|
|
337
|
+
Applied to public ``VisusBot`` action methods (``click_text``,
|
|
338
|
+
``click_image``, ``type_text``, …). When the call happens inside a
|
|
339
|
+
``with bot.start_task_session()`` scope, the result is forwarded to
|
|
340
|
+
:meth:`VisusBot._emit_direct_call_event` so a parallel
|
|
341
|
+
:class:`Session.start_recording` zip contains the same event line +
|
|
342
|
+
per-step / failure screenshot it would have captured via
|
|
343
|
+
:meth:`execute_tasks`. Outside a session the wrapper is transparent.
|
|
344
|
+
"""
|
|
345
|
+
def decorator(method):
|
|
346
|
+
@functools.wraps(method)
|
|
347
|
+
def wrapper(self, *args, **kwargs):
|
|
348
|
+
if not getattr(self, "task_session_active", False):
|
|
349
|
+
return method(self, *args, **kwargs)
|
|
350
|
+
|
|
351
|
+
started = time.monotonic()
|
|
352
|
+
try:
|
|
353
|
+
result = method(self, *args, **kwargs)
|
|
354
|
+
except Exception as exc:
|
|
355
|
+
self._emit_direct_call_event(
|
|
356
|
+
_build_event_task(type_field, method, args, kwargs),
|
|
357
|
+
success=False,
|
|
358
|
+
started_at=started,
|
|
359
|
+
error=str(exc),
|
|
360
|
+
)
|
|
361
|
+
raise
|
|
362
|
+
|
|
363
|
+
success = bool(result) if isinstance(result, bool) else True
|
|
364
|
+
self._emit_direct_call_event(
|
|
365
|
+
_build_event_task(type_field, method, args, kwargs),
|
|
366
|
+
success=success,
|
|
367
|
+
started_at=started,
|
|
368
|
+
error=None if success else f"{method.__name__} returned falsy",
|
|
369
|
+
)
|
|
370
|
+
return result
|
|
371
|
+
return wrapper
|
|
372
|
+
return decorator
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _build_event_task(type_field, method, args, kwargs):
|
|
376
|
+
"""Construct the task dict consumed by ``_emit_task_event``.
|
|
377
|
+
|
|
378
|
+
Maps the first positional arg to a sensible task field based on
|
|
379
|
+
``type_field`` (so ``click_text("Login")`` ends up as
|
|
380
|
+
``{"type": "text", "text": "Login"}``), and folds in any recognized
|
|
381
|
+
kwargs that add useful context to the event.
|
|
382
|
+
"""
|
|
383
|
+
task = {"type": type_field}
|
|
384
|
+
if args:
|
|
385
|
+
primary_key = {
|
|
386
|
+
"text": "text",
|
|
387
|
+
"image": "image",
|
|
388
|
+
"click": "x", # click_at / click_coordinates -- args[0] is x or location
|
|
389
|
+
"relative_image": "anchor_image",
|
|
390
|
+
"type_text": "text",
|
|
391
|
+
"keyboard_command": "command",
|
|
392
|
+
}.get(type_field)
|
|
393
|
+
if primary_key:
|
|
394
|
+
task[primary_key] = args[0]
|
|
395
|
+
else:
|
|
396
|
+
task["arg"] = args[0]
|
|
397
|
+
for key in _RECORDED_KWARGS:
|
|
398
|
+
if key in kwargs and key not in task:
|
|
399
|
+
task[key] = kwargs[key]
|
|
400
|
+
return task
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class VisusBot(CapabilitiesMixin, KeyboardMixin):
|
|
404
|
+
"""High-level GUI automation: native control, image, OCR, coordinates.
|
|
405
|
+
|
|
406
|
+
``VisusBot`` is the single entry point users typically import. It bundles
|
|
407
|
+
the smart-cascade finder (native → image → OCR → coordinates), a
|
|
408
|
+
keyboard/mouse driver (physical or virtual), an OCR engine, and a
|
|
409
|
+
click-through visual overlay for debugging. Every public method is also
|
|
410
|
+
exposed via the ``visus`` CLI and the ``visus-mcp`` MCP server.
|
|
411
|
+
|
|
412
|
+
Built-in capabilities (see method docstrings for full signatures):
|
|
413
|
+
|
|
414
|
+
**Finding things on screen**
|
|
415
|
+
|
|
416
|
+
* :meth:`find_text` / :meth:`click_text` -- OCR-based text targeting
|
|
417
|
+
with confidence threshold, character-class filter, and per-region
|
|
418
|
+
search.
|
|
419
|
+
* :meth:`find_image` / :meth:`click_image` -- Template matching with
|
|
420
|
+
``cv2.matchTemplate`` (falls back to pyscreeze); supports
|
|
421
|
+
scale-variation search when ``specific=False``.
|
|
422
|
+
* :meth:`find_relative_image` -- Find a target image near an anchor
|
|
423
|
+
(good for picking the right row in a repeating grid).
|
|
424
|
+
* :meth:`hybrid_find` -- Try the smart cascade in order until one
|
|
425
|
+
method locates the target.
|
|
426
|
+
|
|
427
|
+
**Acting on things**
|
|
428
|
+
|
|
429
|
+
* :meth:`click_at` -- Click absolute (x, y, w, h) coordinates.
|
|
430
|
+
* :meth:`type_text` -- Type a string into the focused field.
|
|
431
|
+
* Mouse extras from :class:`CapabilitiesMixin`: :meth:`drag`,
|
|
432
|
+
:meth:`triple_click`, :meth:`scroll`, :meth:`move_random`.
|
|
433
|
+
* Keyboard from :class:`KeyboardMixin`: hotkeys, key presses,
|
|
434
|
+
special-sequence parsing (``{ctrl}a{del}{tab}``).
|
|
435
|
+
|
|
436
|
+
**Robustness primitives**
|
|
437
|
+
|
|
438
|
+
* Backtrack: pass ``backtrack=True`` to retry failed steps and
|
|
439
|
+
re-walk the recent action history (see
|
|
440
|
+
:meth:`start_task_session`).
|
|
441
|
+
* Waits: ``wait_until_found`` / ``wait_until_disappears`` /
|
|
442
|
+
``wait_timeout`` on find/click methods.
|
|
443
|
+
* Failure artifacts: when ``screenshot_on_failure=True`` is set in
|
|
444
|
+
the config, every uncaught failure dumps a PNG + JSON blob into
|
|
445
|
+
``artifacts_dir``.
|
|
446
|
+
|
|
447
|
+
**Overlay (visual feedback)**
|
|
448
|
+
|
|
449
|
+
* :meth:`configure_overlay`, :meth:`get_overlay_config`,
|
|
450
|
+
:attr:`show_overlay`. On Windows the overlay is a layered
|
|
451
|
+
click-through window (``WS_EX_TRANSPARENT``), so the click that
|
|
452
|
+
follows lands on the host below regardless of tech (native,
|
|
453
|
+
Java, Electron, browsers).
|
|
454
|
+
|
|
455
|
+
Examples:
|
|
456
|
+
Defaults are usually fine; install Tesseract and go::
|
|
457
|
+
|
|
458
|
+
from visus.desktop import VisusBot, ensure_window
|
|
459
|
+
ensure_window(process="notepad.exe")
|
|
460
|
+
bot = VisusBot()
|
|
461
|
+
bot.click_text("File")
|
|
462
|
+
bot.type_text("hello world")
|
|
463
|
+
|
|
464
|
+
Tune via a dict (delegated to :class:`BotVisionConfig`)::
|
|
465
|
+
|
|
466
|
+
bot = VisusBot(config={
|
|
467
|
+
"confidence_threshold": 80.0,
|
|
468
|
+
"overlay_color": "green",
|
|
469
|
+
"matcher": "cv2",
|
|
470
|
+
})
|
|
471
|
+
|
|
472
|
+
Or build the config object yourself for finer control::
|
|
473
|
+
|
|
474
|
+
from visus.desktop import BotVisionConfig
|
|
475
|
+
cfg = BotVisionConfig({"show_overlay": False})
|
|
476
|
+
bot = VisusBot(config=cfg)
|
|
477
|
+
"""
|
|
478
|
+
|
|
479
|
+
def __init__(self, config=None):
|
|
480
|
+
"""Initialize Visus Desktop.
|
|
481
|
+
|
|
482
|
+
Wires the executor, OCR engine, image processor, and overlay
|
|
483
|
+
master toggle. Construction is cheap; Tesseract is auto-detected only
|
|
484
|
+
when ``ocr_backend == "tesseract"`` (the default backend is now
|
|
485
|
+
``rapidocr``, which uses bundled models and skips detection).
|
|
486
|
+
|
|
487
|
+
Args:
|
|
488
|
+
config: Custom settings. Three forms are accepted:
|
|
489
|
+
|
|
490
|
+
* ``None`` (default) -- use the library defaults.
|
|
491
|
+
* ``dict`` -- a mapping of any subset of the keys
|
|
492
|
+
documented on :class:`BotVisionConfig` (hover that class
|
|
493
|
+
for the full reference of every key, type, and default).
|
|
494
|
+
Common ones:
|
|
495
|
+
|
|
496
|
+
- ``confidence_threshold`` (float, ``75.0``): OCR cutoff.
|
|
497
|
+
- ``retry_attempts`` (int, ``3``): default retries.
|
|
498
|
+
- ``default_delay`` (float, ``1.0``): pause between
|
|
499
|
+
actions (s).
|
|
500
|
+
- ``show_overlay`` (bool, ``True``): master toggle for
|
|
501
|
+
the click-through visual overlay.
|
|
502
|
+
- ``overlay_color`` (str, ``"red"``) /
|
|
503
|
+
``overlay_duration`` (int ms, ``1000``) /
|
|
504
|
+
``overlay_width`` (int px, ``4``): overlay look.
|
|
505
|
+
- ``default_mouse_button`` (str, ``"left"``): one of
|
|
506
|
+
``"left"``, ``"right"``, ``"double"``, ``"move_to"``.
|
|
507
|
+
- ``use_virtual_mouse`` (bool, ``False``): click without
|
|
508
|
+
moving the physical cursor (Windows, PostMessage).
|
|
509
|
+
- ``matcher`` (str, ``"cv2"``): ``"cv2"`` or
|
|
510
|
+
``"pyscreeze"``.
|
|
511
|
+
- ``ocr_backend`` (str, ``"rapidocr"``): ``"rapidocr"``
|
|
512
|
+
(default) or ``"tesseract"``.
|
|
513
|
+
- ``ocr_languages`` (list[str], ``["eng"]``).
|
|
514
|
+
- ``wait_timeout`` (int s, ``30``): default wait timeout.
|
|
515
|
+
- ``log_level`` (str, ``"INFO"``) /
|
|
516
|
+
``log_format`` (str, ``"text"`` or ``"json"``).
|
|
517
|
+
- ``artifacts_dir`` (str, ``"./visus-artifacts"``) /
|
|
518
|
+
``screenshot_on_failure`` (bool, ``True``).
|
|
519
|
+
|
|
520
|
+
* :class:`BotVisionConfig` -- a pre-built config object,
|
|
521
|
+
used as-is. Use this when you want to reuse the same
|
|
522
|
+
config across several bots or when you have already
|
|
523
|
+
validated it.
|
|
524
|
+
|
|
525
|
+
Examples:
|
|
526
|
+
>>> bot = VisusBot() # defaults
|
|
527
|
+
>>> bot = VisusBot(config={"overlay_color": "green"})
|
|
528
|
+
>>> from visus.desktop import BotVisionConfig
|
|
529
|
+
>>> cfg = BotVisionConfig({"show_overlay": False})
|
|
530
|
+
>>> bot = VisusBot(config=cfg)
|
|
531
|
+
"""
|
|
532
|
+
if isinstance(config, dict):
|
|
533
|
+
self.config = BotVisionConfig(config)
|
|
534
|
+
elif isinstance(config, BotVisionConfig):
|
|
535
|
+
self.config = config
|
|
536
|
+
else:
|
|
537
|
+
self.config = get_default_config()
|
|
538
|
+
|
|
539
|
+
# Bind the hybrid finder lazily so that monkeypatches on
|
|
540
|
+
# ``self.hybrid_find`` (e.g. in tests) take effect for hybrid
|
|
541
|
+
# task dispatch in the executor.
|
|
542
|
+
self.executor = TaskExecutor(
|
|
543
|
+
self.config,
|
|
544
|
+
hybrid_finder=lambda name, **kw: self.hybrid_find(name, **kw),
|
|
545
|
+
)
|
|
546
|
+
self.ocr_engine = OCREngine(self.config)
|
|
547
|
+
self.image_processor = ImageProcessor()
|
|
548
|
+
|
|
549
|
+
# Overlay master toggle - accessible via the ``show_overlay`` property.
|
|
550
|
+
self._show_overlay = self.config.get("show_overlay", True)
|
|
551
|
+
|
|
552
|
+
# Backtrack system for individual methods
|
|
553
|
+
self.individual_task_history = [] # History of executed methods
|
|
554
|
+
self.backtrack_enabled_globally = False # Whether it is in backtrack mode
|
|
555
|
+
self.current_task_index = -1 # Index of the current task
|
|
556
|
+
self.max_backtrack_attempts = 2 # Maximum backtrack attempts per task
|
|
557
|
+
self.task_session_active = False # Whether it is in an individual task session
|
|
558
|
+
|
|
559
|
+
def _add_to_task_history(self, method_name, args, kwargs):
|
|
560
|
+
"""Add a method to the individual task history."""
|
|
561
|
+
task_info = {
|
|
562
|
+
"method": method_name,
|
|
563
|
+
"args": args,
|
|
564
|
+
"kwargs": kwargs,
|
|
565
|
+
"index": len(self.individual_task_history),
|
|
566
|
+
"success": None,
|
|
567
|
+
"backtrack_attempts": 0,
|
|
568
|
+
"timestamp": time.time(),
|
|
569
|
+
}
|
|
570
|
+
self.individual_task_history.append(task_info)
|
|
571
|
+
self.current_task_index = len(self.individual_task_history) - 1
|
|
572
|
+
return self.current_task_index
|
|
573
|
+
|
|
574
|
+
def start_task_session(self):
|
|
575
|
+
"""Open a manual task session.
|
|
576
|
+
|
|
577
|
+
While a session is active, direct calls like :meth:`click_text` and
|
|
578
|
+
:meth:`click_image` emit the same structured ``task_event`` as
|
|
579
|
+
:meth:`execute_tasks` so they show up in a
|
|
580
|
+
:class:`~visus.desktop.core.session.Session` recording (events +
|
|
581
|
+
per-step screenshots in the zip). Also enables the backtrack
|
|
582
|
+
history for retries between methods.
|
|
583
|
+
"""
|
|
584
|
+
import uuid
|
|
585
|
+
|
|
586
|
+
self.task_session_active = True
|
|
587
|
+
self.individual_task_history = []
|
|
588
|
+
self.current_task_index = -1
|
|
589
|
+
# Tag every event emitted during this session with a stable run id
|
|
590
|
+
# so the resulting events.jsonl groups cleanly in reports.
|
|
591
|
+
self.executor.current_run_id = uuid.uuid4().hex[:8]
|
|
592
|
+
logger.debug(
|
|
593
|
+
"Task session started (run_id=%s) -- direct calls will be recorded",
|
|
594
|
+
self.executor.current_run_id,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
def end_task_session(self):
|
|
598
|
+
"""Close the manual task session opened by :meth:`start_task_session`."""
|
|
599
|
+
self.task_session_active = False
|
|
600
|
+
successful_tasks = sum(
|
|
601
|
+
1 for task in self.individual_task_history if task.get("success", False)
|
|
602
|
+
)
|
|
603
|
+
total_tasks = len(self.individual_task_history)
|
|
604
|
+
logger.debug(
|
|
605
|
+
"Task session ended: %d/%d successful tasks (run_id=%s)",
|
|
606
|
+
successful_tasks, total_tasks, self.executor.current_run_id,
|
|
607
|
+
)
|
|
608
|
+
self.executor.current_run_id = None
|
|
609
|
+
return successful_tasks, total_tasks
|
|
610
|
+
|
|
611
|
+
def _emit_direct_call_event(
|
|
612
|
+
self, task_for_event, success, started_at, location=None, error=None
|
|
613
|
+
):
|
|
614
|
+
"""Emit a structured ``task_event`` for a direct method call.
|
|
615
|
+
|
|
616
|
+
No-op unless a task session is open (so direct calls outside a
|
|
617
|
+
session keep their original "no zip noise" behavior). Mirrors the
|
|
618
|
+
shape of :meth:`TaskExecutor._emit_task_event`, including the
|
|
619
|
+
success / failure screenshot decision driven by
|
|
620
|
+
``screenshot_steps`` and ``screenshot_on_failure``.
|
|
621
|
+
"""
|
|
622
|
+
if not getattr(self, "task_session_active", False):
|
|
623
|
+
return
|
|
624
|
+
if self.config.get("log_format", "text") != "json":
|
|
625
|
+
return
|
|
626
|
+
|
|
627
|
+
task_index = len(self.individual_task_history)
|
|
628
|
+
self.individual_task_history.append({
|
|
629
|
+
"method": task_for_event.get("type") or task_for_event.get("text") or "direct",
|
|
630
|
+
"success": bool(success),
|
|
631
|
+
})
|
|
632
|
+
|
|
633
|
+
target = (
|
|
634
|
+
task_for_event.get("text")
|
|
635
|
+
or task_for_event.get("image")
|
|
636
|
+
or task_for_event.get("type", "direct_call")
|
|
637
|
+
)
|
|
638
|
+
# If the caller (decorator) didn't pass a location, fall back to
|
|
639
|
+
# the one stashed by ``TaskExecutor._perform_action`` -- that's
|
|
640
|
+
# what direct ``click_text`` / ``click_image`` calls landed on,
|
|
641
|
+
# so the screenshot can be annotated even though the public
|
|
642
|
+
# method only returned bool.
|
|
643
|
+
effective_location = location or getattr(
|
|
644
|
+
self.executor, "last_action_location", None
|
|
645
|
+
)
|
|
646
|
+
result = TaskResult(task_index, bool(success), str(target), effective_location)
|
|
647
|
+
result.attempts = 1
|
|
648
|
+
result.duration_ms = int((time.monotonic() - started_at) * 1000)
|
|
649
|
+
result.error = error
|
|
650
|
+
if success and self.config.get("screenshot_steps"):
|
|
651
|
+
result.step_screenshot = self.executor._capture_step_screenshot(
|
|
652
|
+
task_index, location=effective_location
|
|
653
|
+
)
|
|
654
|
+
elif not success and self.config.get("screenshot_on_failure", True):
|
|
655
|
+
result.failure_screenshot = self.executor._save_failure_screenshot(
|
|
656
|
+
task_index, location=effective_location
|
|
657
|
+
)
|
|
658
|
+
self.executor._emit_task_event(result, task_for_event)
|
|
659
|
+
|
|
660
|
+
# Overlay control properties
|
|
661
|
+
@property
|
|
662
|
+
def show_overlay(self):
|
|
663
|
+
"""Control whether the visual overlay is displayed before clicks."""
|
|
664
|
+
return self._show_overlay
|
|
665
|
+
|
|
666
|
+
@show_overlay.setter
|
|
667
|
+
def show_overlay(self, value):
|
|
668
|
+
"""Set whether the visual overlay is displayed before clicks."""
|
|
669
|
+
self._show_overlay = bool(value)
|
|
670
|
+
# Also update the configuration
|
|
671
|
+
self.config.config["show_overlay"] = self._show_overlay
|
|
672
|
+
|
|
673
|
+
def configure_overlay(self, enabled=None, color=None, duration=None, width=None):
|
|
674
|
+
"""
|
|
675
|
+
Conveniently configure the overlay parameters.
|
|
676
|
+
|
|
677
|
+
Args:
|
|
678
|
+
enabled (bool, optional): Whether the overlay is enabled
|
|
679
|
+
color (str, optional): Overlay color - options: 'red', 'blue', 'green', 'yellow',
|
|
680
|
+
'purple', 'orange', 'cyan', 'magenta', 'white', 'black'
|
|
681
|
+
duration (int, optional): Duration in milliseconds (500-5000 recommended)
|
|
682
|
+
width (int, optional): Overlay line width (1-10 recommended)
|
|
683
|
+
|
|
684
|
+
Examples:
|
|
685
|
+
>>> bot = VisusBot()
|
|
686
|
+
>>> bot.configure_overlay(enabled=True, color="blue", duration=2000)
|
|
687
|
+
>>> bot.configure_overlay(color="green", width=6)
|
|
688
|
+
|
|
689
|
+
Raises:
|
|
690
|
+
ValueError: If an invalid color is provided
|
|
691
|
+
"""
|
|
692
|
+
valid_colors = [
|
|
693
|
+
"red",
|
|
694
|
+
"blue",
|
|
695
|
+
"green",
|
|
696
|
+
"yellow",
|
|
697
|
+
"purple",
|
|
698
|
+
"orange",
|
|
699
|
+
"cyan",
|
|
700
|
+
"magenta",
|
|
701
|
+
"white",
|
|
702
|
+
"black",
|
|
703
|
+
]
|
|
704
|
+
|
|
705
|
+
if enabled is not None:
|
|
706
|
+
self.show_overlay = enabled
|
|
707
|
+
|
|
708
|
+
if color is not None:
|
|
709
|
+
if color.lower() not in [c.lower() for c in valid_colors]:
|
|
710
|
+
raise ValueError(
|
|
711
|
+
f"Invalid color '{color}'. Available colors: {', '.join(valid_colors)}"
|
|
712
|
+
)
|
|
713
|
+
self.config.config["overlay_color"] = color.lower()
|
|
714
|
+
|
|
715
|
+
if duration is not None:
|
|
716
|
+
if not isinstance(duration, (int, float)) or duration <= 0:
|
|
717
|
+
raise ValueError(
|
|
718
|
+
f"Duration must be a positive number, received: {duration}"
|
|
719
|
+
)
|
|
720
|
+
if duration > 10000:
|
|
721
|
+
logger.warning(
|
|
722
|
+
f"Duration too high ({duration}ms). Consider using less than 5000ms."
|
|
723
|
+
)
|
|
724
|
+
self.config.config["overlay_duration"] = int(duration)
|
|
725
|
+
|
|
726
|
+
if width is not None:
|
|
727
|
+
if not isinstance(width, (int, float)) or width <= 0:
|
|
728
|
+
raise ValueError(f"Width must be a positive number, received: {width}")
|
|
729
|
+
if width > 15:
|
|
730
|
+
logger.warning(
|
|
731
|
+
f"Width too high ({width}). Consider using less than 10."
|
|
732
|
+
)
|
|
733
|
+
self.config.config["overlay_width"] = int(width)
|
|
734
|
+
|
|
735
|
+
def get_overlay_config(self):
|
|
736
|
+
"""
|
|
737
|
+
Return the current overlay configuration.
|
|
738
|
+
|
|
739
|
+
Returns:
|
|
740
|
+
dict: Overlay settings
|
|
741
|
+
"""
|
|
742
|
+
return {
|
|
743
|
+
"enabled": self.show_overlay,
|
|
744
|
+
"color": self.config.config.get("overlay_color", "red"),
|
|
745
|
+
"duration": self.config.config.get("overlay_duration", 1000),
|
|
746
|
+
"width": self.config.config.get("overlay_width", 4),
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
@staticmethod
|
|
750
|
+
def get_available_overlay_colors():
|
|
751
|
+
"""
|
|
752
|
+
Return the list of available overlay colors.
|
|
753
|
+
|
|
754
|
+
Returns:
|
|
755
|
+
list: List of available colors
|
|
756
|
+
|
|
757
|
+
Examples:
|
|
758
|
+
>>> colors = VisusBot.get_available_overlay_colors()
|
|
759
|
+
>>> print("Available colors:", colors)
|
|
760
|
+
"""
|
|
761
|
+
return [
|
|
762
|
+
"red",
|
|
763
|
+
"blue",
|
|
764
|
+
"green",
|
|
765
|
+
"yellow",
|
|
766
|
+
"purple",
|
|
767
|
+
"orange",
|
|
768
|
+
"cyan",
|
|
769
|
+
"magenta",
|
|
770
|
+
"white",
|
|
771
|
+
"black",
|
|
772
|
+
]
|
|
773
|
+
|
|
774
|
+
def test_overlay_colors(self, duration=1500):
|
|
775
|
+
"""
|
|
776
|
+
Test all available overlay colors.
|
|
777
|
+
|
|
778
|
+
Args:
|
|
779
|
+
duration (int): Duration of each color in milliseconds
|
|
780
|
+
|
|
781
|
+
Examples:
|
|
782
|
+
>>> bot = VisusBot()
|
|
783
|
+
>>> bot.test_overlay_colors() # Shows each color for 1.5 seconds
|
|
784
|
+
"""
|
|
785
|
+
from .core.overlay import VisualOverlay
|
|
786
|
+
|
|
787
|
+
colors = self.get_available_overlay_colors()
|
|
788
|
+
test_region = (400, 300, 200, 100) # Center of the screen
|
|
789
|
+
|
|
790
|
+
print("🎨 Testing overlay colors...")
|
|
791
|
+
print("📍 Look at the center of the screen!")
|
|
792
|
+
|
|
793
|
+
for i, color in enumerate(colors, 1):
|
|
794
|
+
print(f" {i}. Color: {color.upper()}")
|
|
795
|
+
overlay = VisualOverlay(color=color, width=6, duration=duration)
|
|
796
|
+
overlay.show(test_region, blocking=True)
|
|
797
|
+
|
|
798
|
+
print("✅ Color test complete!")
|
|
799
|
+
|
|
800
|
+
def _execute_with_individual_backtrack(
|
|
801
|
+
self, method_name, method_func, *args, **kwargs
|
|
802
|
+
):
|
|
803
|
+
"""
|
|
804
|
+
Execute an individual method with backtrack capability.
|
|
805
|
+
|
|
806
|
+
Args:
|
|
807
|
+
method_name (str): Name of the method ('click_image', 'click_text', etc.)
|
|
808
|
+
method_func (callable): The method function to execute
|
|
809
|
+
*args: Positional arguments
|
|
810
|
+
**kwargs: Keyword arguments
|
|
811
|
+
|
|
812
|
+
Returns:
|
|
813
|
+
bool: Result of the execution
|
|
814
|
+
"""
|
|
815
|
+
backtrack = kwargs.get("backtrack", False)
|
|
816
|
+
|
|
817
|
+
# If backtrack is not enabled or there is no active session, execute normally
|
|
818
|
+
if not backtrack or not self.task_session_active:
|
|
819
|
+
if backtrack and not self.task_session_active:
|
|
820
|
+
# Auto-start a session if backtrack=True but no session is active
|
|
821
|
+
self.start_task_session()
|
|
822
|
+
|
|
823
|
+
# Remove backtrack from kwargs to avoid infinite recursion
|
|
824
|
+
execution_kwargs = kwargs.copy()
|
|
825
|
+
execution_kwargs.pop("backtrack", None)
|
|
826
|
+
return method_func(*args, **execution_kwargs)
|
|
827
|
+
|
|
828
|
+
# Add to the history
|
|
829
|
+
task_index = self._add_to_task_history(method_name, args, kwargs)
|
|
830
|
+
|
|
831
|
+
logger.info(
|
|
832
|
+
f"🎯 Running {method_name} (task {task_index + 1}) with backtrack enabled"
|
|
833
|
+
)
|
|
834
|
+
|
|
835
|
+
# Remove backtrack from kwargs to avoid infinite recursion
|
|
836
|
+
execution_kwargs = kwargs.copy()
|
|
837
|
+
execution_kwargs.pop("backtrack", None)
|
|
838
|
+
|
|
839
|
+
# Try to execute the method
|
|
840
|
+
success = method_func(*args, **execution_kwargs)
|
|
841
|
+
|
|
842
|
+
# Update the result in the history
|
|
843
|
+
self.individual_task_history[task_index]["success"] = success
|
|
844
|
+
|
|
845
|
+
if success:
|
|
846
|
+
logger.info(
|
|
847
|
+
f"✓ {method_name} (task {task_index + 1}) executed successfully!"
|
|
848
|
+
)
|
|
849
|
+
return True
|
|
850
|
+
else:
|
|
851
|
+
# Failed - check whether a backtrack is possible
|
|
852
|
+
if task_index > 0: # There are previous tasks
|
|
853
|
+
current_task = self.individual_task_history[task_index]
|
|
854
|
+
|
|
855
|
+
if current_task["backtrack_attempts"] < self.max_backtrack_attempts:
|
|
856
|
+
current_task["backtrack_attempts"] += 1
|
|
857
|
+
|
|
858
|
+
# Execute the previous task
|
|
859
|
+
prev_task = self.individual_task_history[task_index - 1]
|
|
860
|
+
logger.info(
|
|
861
|
+
f"🔄 BACKTRACK: {method_name} (task {task_index + 1}) failed. "
|
|
862
|
+
f"Re-running the previous task ({prev_task['method']}) and trying again..."
|
|
863
|
+
)
|
|
864
|
+
|
|
865
|
+
# Re-run the previous task
|
|
866
|
+
prev_method_name = prev_task["method"]
|
|
867
|
+
prev_method_func = getattr(self, f"_{prev_method_name}_internal")
|
|
868
|
+
prev_kwargs = prev_task["kwargs"].copy()
|
|
869
|
+
prev_kwargs.pop(
|
|
870
|
+
"backtrack", None
|
|
871
|
+
) # Remove backtrack to avoid recursion
|
|
872
|
+
|
|
873
|
+
prev_success = prev_method_func(*prev_task["args"], **prev_kwargs)
|
|
874
|
+
self.individual_task_history[task_index - 1]["success"] = (
|
|
875
|
+
prev_success
|
|
876
|
+
)
|
|
877
|
+
|
|
878
|
+
if prev_success:
|
|
879
|
+
logger.info(
|
|
880
|
+
f"✓ Previous task ({prev_task['method']}) re-run successfully"
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
# Wait a bit and try the current task again
|
|
884
|
+
time.sleep(0.5)
|
|
885
|
+
logger.info(
|
|
886
|
+
f"🔄 Retrying {method_name} (task {task_index + 1}) after backtrack..."
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
retry_success = method_func(*args, **execution_kwargs)
|
|
890
|
+
self.individual_task_history[task_index]["success"] = (
|
|
891
|
+
retry_success
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
if retry_success:
|
|
895
|
+
logger.info(
|
|
896
|
+
f"✓ {method_name} (task {task_index + 1}) succeeded after backtrack!"
|
|
897
|
+
)
|
|
898
|
+
return True
|
|
899
|
+
else:
|
|
900
|
+
logger.warning(
|
|
901
|
+
f"✗ {method_name} (task {task_index + 1}) still failed after backtrack"
|
|
902
|
+
)
|
|
903
|
+
else:
|
|
904
|
+
logger.warning(
|
|
905
|
+
f"✗ Previous task ({prev_task['method']}) also failed on re-run"
|
|
906
|
+
)
|
|
907
|
+
else:
|
|
908
|
+
logger.warning(
|
|
909
|
+
f"✗ {method_name} (task {task_index + 1}) failed after {self.max_backtrack_attempts} backtrack attempts"
|
|
910
|
+
)
|
|
911
|
+
else:
|
|
912
|
+
logger.warning(
|
|
913
|
+
f"✗ {method_name} (task {task_index + 1}) failed but it is the first task (no backtrack possible)"
|
|
914
|
+
)
|
|
915
|
+
|
|
916
|
+
return False
|
|
917
|
+
|
|
918
|
+
# ------------------------------------------------------------------
|
|
919
|
+
# Hybrid targeting (Phase 3 / T1)
|
|
920
|
+
# ------------------------------------------------------------------
|
|
921
|
+
|
|
922
|
+
_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".bmp", ".gif", ".tiff")
|
|
923
|
+
|
|
924
|
+
# Control types queried (in order) before falling back to an
|
|
925
|
+
# unfiltered descendants() walk. Filtering by control_type at the UIA
|
|
926
|
+
# level is far cheaper than pulling the whole accessibility tree, so
|
|
927
|
+
# clickable elements (menu items, buttons, tabs, links) are found fast.
|
|
928
|
+
_NATIVE_CONTROL_TYPES = (
|
|
929
|
+
"MenuItem",
|
|
930
|
+
"Button",
|
|
931
|
+
"TabItem",
|
|
932
|
+
"ListItem",
|
|
933
|
+
"Hyperlink",
|
|
934
|
+
"Text",
|
|
935
|
+
"Edit",
|
|
936
|
+
"CheckBox",
|
|
937
|
+
"RadioButton",
|
|
938
|
+
"ComboBox",
|
|
939
|
+
)
|
|
940
|
+
|
|
941
|
+
def _hybrid_native_lookup(self, name, region=None, native_timeout: float = 5.0):
|
|
942
|
+
"""Best-effort native UIA lookup used by :meth:`hybrid_find`.
|
|
943
|
+
|
|
944
|
+
Searches the foreground window's UIA tree (via ``pywinauto.Desktop``)
|
|
945
|
+
for a control whose **title**, **automation_id**, or **control_type**
|
|
946
|
+
contains ``name`` (case-insensitive). Returns an ``(x, y, w, h)`` box
|
|
947
|
+
or ``None`` when not found.
|
|
948
|
+
|
|
949
|
+
Two safeguards keep UIA's notoriously slow tree enumeration from
|
|
950
|
+
hanging the caller:
|
|
951
|
+
|
|
952
|
+
* **control_type filtering** -- cheap ``descendants(control_type=...)``
|
|
953
|
+
queries for common clickable types run first; the expensive
|
|
954
|
+
unfiltered walk is only the last resort.
|
|
955
|
+
* **``native_timeout``** -- the whole lookup runs in a worker thread
|
|
956
|
+
and is abandoned (returns ``None``) if it exceeds the timeout, so
|
|
957
|
+
the hybrid chain falls back to OCR promptly instead of blocking.
|
|
958
|
+
|
|
959
|
+
On non-Windows / missing pywinauto it returns ``None`` cleanly.
|
|
960
|
+
"""
|
|
961
|
+
try:
|
|
962
|
+
import pywinauto # type: ignore[import-not-found] # noqa: F401
|
|
963
|
+
except ImportError:
|
|
964
|
+
return None
|
|
965
|
+
|
|
966
|
+
result: dict = {"box": None}
|
|
967
|
+
|
|
968
|
+
def _work() -> None:
|
|
969
|
+
try:
|
|
970
|
+
import pywinauto as _pwa
|
|
971
|
+
|
|
972
|
+
desktop = _pwa.Desktop(backend="uia")
|
|
973
|
+
top = desktop.window(active_only=True)
|
|
974
|
+
if not top.exists(timeout=0.5):
|
|
975
|
+
return
|
|
976
|
+
needle = (name or "").lower()
|
|
977
|
+
if not needle:
|
|
978
|
+
return
|
|
979
|
+
|
|
980
|
+
def _match(descendant):
|
|
981
|
+
try:
|
|
982
|
+
info = descendant.element_info
|
|
983
|
+
title = (info.name or "").lower()
|
|
984
|
+
auto_id = (getattr(info, "automation_id", "") or "").lower()
|
|
985
|
+
ctrl = (getattr(info, "control_type", "") or "").lower()
|
|
986
|
+
except Exception:
|
|
987
|
+
return None
|
|
988
|
+
if needle in title or needle in auto_id or needle in ctrl:
|
|
989
|
+
try:
|
|
990
|
+
r = descendant.rectangle()
|
|
991
|
+
except Exception:
|
|
992
|
+
return None
|
|
993
|
+
return (
|
|
994
|
+
int(r.left),
|
|
995
|
+
int(r.top),
|
|
996
|
+
int(r.right - r.left),
|
|
997
|
+
int(r.bottom - r.top),
|
|
998
|
+
)
|
|
999
|
+
return None
|
|
1000
|
+
|
|
1001
|
+
# Cheap filtered passes first, then the unfiltered fallback.
|
|
1002
|
+
queries = [
|
|
1003
|
+
{"control_type": ct} for ct in self._NATIVE_CONTROL_TYPES
|
|
1004
|
+
]
|
|
1005
|
+
queries.append({}) # unfiltered last resort
|
|
1006
|
+
for kwargs in queries:
|
|
1007
|
+
try:
|
|
1008
|
+
candidates = top.descendants(**kwargs)
|
|
1009
|
+
except Exception:
|
|
1010
|
+
continue
|
|
1011
|
+
for descendant in candidates:
|
|
1012
|
+
box = _match(descendant)
|
|
1013
|
+
if box is not None:
|
|
1014
|
+
result["box"] = box
|
|
1015
|
+
return
|
|
1016
|
+
except Exception as exc: # any pywinauto-level fault is non-fatal
|
|
1017
|
+
logger.debug("native hybrid lookup failed: %s", exc)
|
|
1018
|
+
|
|
1019
|
+
import threading
|
|
1020
|
+
|
|
1021
|
+
worker = threading.Thread(target=_work, daemon=True)
|
|
1022
|
+
worker.start()
|
|
1023
|
+
worker.join(timeout=native_timeout)
|
|
1024
|
+
if worker.is_alive():
|
|
1025
|
+
logger.debug(
|
|
1026
|
+
"native lookup exceeded %.1fs; giving up (falling back to OCR)",
|
|
1027
|
+
native_timeout,
|
|
1028
|
+
)
|
|
1029
|
+
return None
|
|
1030
|
+
return result["box"]
|
|
1031
|
+
|
|
1032
|
+
def hybrid_find(self, name, region=None):
|
|
1033
|
+
"""Find a target by trying native → OCR → image in order.
|
|
1034
|
+
|
|
1035
|
+
Each layer is mockable; the first hit wins. Returns an
|
|
1036
|
+
``(x, y, w, h)`` tuple or ``None``. The image layer is only
|
|
1037
|
+
attempted when ``name`` looks like an image filename (one of
|
|
1038
|
+
``.png .jpg .jpeg .bmp .gif .tiff``).
|
|
1039
|
+
"""
|
|
1040
|
+
# Layer 1 -- native UI handle (pywinauto).
|
|
1041
|
+
try:
|
|
1042
|
+
box = self._hybrid_native_lookup(name, region=region)
|
|
1043
|
+
except Exception:
|
|
1044
|
+
box = None
|
|
1045
|
+
if box:
|
|
1046
|
+
return box
|
|
1047
|
+
|
|
1048
|
+
# Layer 2 -- OCR.
|
|
1049
|
+
box = self.find_text(name, region=region)
|
|
1050
|
+
if box:
|
|
1051
|
+
return box
|
|
1052
|
+
|
|
1053
|
+
# Layer 3 -- image template (only for paths that look like images).
|
|
1054
|
+
lower = name.lower() if isinstance(name, str) else ""
|
|
1055
|
+
if lower.endswith(self._IMAGE_SUFFIXES):
|
|
1056
|
+
box = self.find_image(name, region=region)
|
|
1057
|
+
if box:
|
|
1058
|
+
return box
|
|
1059
|
+
return None
|
|
1060
|
+
|
|
1061
|
+
def execute_tasks(self, tasks, resume: bool = False):
|
|
1062
|
+
"""
|
|
1063
|
+
Execute a list of tasks sequentially.
|
|
1064
|
+
|
|
1065
|
+
Args:
|
|
1066
|
+
tasks (list): List of dictionaries with task settings
|
|
1067
|
+
resume (bool): When True, skip indices already completed in a
|
|
1068
|
+
previous interrupted run (Phase 2 / R3). The checkpoint
|
|
1069
|
+
file ``<artifacts_dir>/checkpoint.json`` is invalidated
|
|
1070
|
+
when the task list changes shape.
|
|
1071
|
+
|
|
1072
|
+
Returns:
|
|
1073
|
+
list: List of TaskResult objects with the result of each task
|
|
1074
|
+
|
|
1075
|
+
Examples:
|
|
1076
|
+
>>> # Basic tasks
|
|
1077
|
+
>>> tasks = [
|
|
1078
|
+
... {'text': 'Login', 'region': (100, 100, 500, 300)},
|
|
1079
|
+
... {'image': 'button.png', 'delay': 2}
|
|
1080
|
+
... ]
|
|
1081
|
+
>>>
|
|
1082
|
+
>>> # Advanced tasks with ALL the new features:
|
|
1083
|
+
>>> advanced_tasks = [
|
|
1084
|
+
... # 1. OCR text search
|
|
1085
|
+
... {
|
|
1086
|
+
... 'text': 'Usuário',
|
|
1087
|
+
... 'region': (100, 100, 500, 300),
|
|
1088
|
+
... 'char_type': 'letters',
|
|
1089
|
+
... 'confidence': 0.8,
|
|
1090
|
+
... 'occurrence': 1,
|
|
1091
|
+
... 'backtrack': True,
|
|
1092
|
+
... 'delay': 1,
|
|
1093
|
+
... 'sendtext': 'admin{tab}password{enter}'
|
|
1094
|
+
... },
|
|
1095
|
+
... # 2. Relative image search (NEW!)
|
|
1096
|
+
... {
|
|
1097
|
+
... 'type': 'relative_image',
|
|
1098
|
+
... 'anchor_image': 'warning_icon.png',
|
|
1099
|
+
... 'target_image': 'ok_button.png',
|
|
1100
|
+
... 'max_distance': 200,
|
|
1101
|
+
... 'confidence': 0.9,
|
|
1102
|
+
... 'target_region': (0, 0, 800, 600),
|
|
1103
|
+
... 'specific': True,
|
|
1104
|
+
... 'backtrack': True,
|
|
1105
|
+
... 'delay': 1
|
|
1106
|
+
... },
|
|
1107
|
+
... # 3. Click on specific coordinates (NEW!)
|
|
1108
|
+
... {
|
|
1109
|
+
... 'type': 'click',
|
|
1110
|
+
... 'x': 100,
|
|
1111
|
+
... 'y': 200,
|
|
1112
|
+
... 'mouse_button': 'right',
|
|
1113
|
+
... 'delay': 0.5,
|
|
1114
|
+
... 'backtrack': False
|
|
1115
|
+
... },
|
|
1116
|
+
... # 4. Text typing (NEW!)
|
|
1117
|
+
... {
|
|
1118
|
+
... 'type': 'type_text',
|
|
1119
|
+
... 'text': 'Hello World!',
|
|
1120
|
+
... 'interval': 0.05,
|
|
1121
|
+
... 'delay': 1
|
|
1122
|
+
... },
|
|
1123
|
+
... # 5. Keyboard command (NEW!)
|
|
1124
|
+
... {
|
|
1125
|
+
... 'type': 'keyboard_command',
|
|
1126
|
+
... 'command': 'Ctrl+S',
|
|
1127
|
+
... 'delay': 1
|
|
1128
|
+
... }
|
|
1129
|
+
... ]
|
|
1130
|
+
>>>
|
|
1131
|
+
>>> results = bot.execute_tasks(advanced_tasks)
|
|
1132
|
+
"""
|
|
1133
|
+
return self.executor.execute_tasks(tasks, resume=resume)
|
|
1134
|
+
|
|
1135
|
+
def find_text(
|
|
1136
|
+
self,
|
|
1137
|
+
text,
|
|
1138
|
+
region=None,
|
|
1139
|
+
filter_type="both",
|
|
1140
|
+
confidence_threshold=75.0,
|
|
1141
|
+
occurrence=1,
|
|
1142
|
+
max_attempts=3,
|
|
1143
|
+
backtrack=False,
|
|
1144
|
+
wait_until_found=False,
|
|
1145
|
+
wait_until_disappears=False,
|
|
1146
|
+
wait_timeout=None,
|
|
1147
|
+
word_box=False,
|
|
1148
|
+
):
|
|
1149
|
+
"""
|
|
1150
|
+
Find text on screen using advanced OCR with all features.
|
|
1151
|
+
|
|
1152
|
+
Args:
|
|
1153
|
+
text (str): Text to find
|
|
1154
|
+
region (tuple, optional): (x, y, width, height) of the search region
|
|
1155
|
+
filter_type (str): Filter type ("numbers", "letters", "both") - equivalent to char_type
|
|
1156
|
+
confidence_threshold (float): Minimum confidence threshold
|
|
1157
|
+
occurrence (int): Which occurrence to look for (1 = first, 2 = second, etc.)
|
|
1158
|
+
max_attempts (int): Maximum number of attempts if backtrack=True
|
|
1159
|
+
backtrack (bool): Whether to try multiple times with adjustments
|
|
1160
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
|
|
1161
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
|
|
1162
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
1163
|
+
|
|
1164
|
+
Returns:
|
|
1165
|
+
tuple: Coordinates (x, y, width, height) where the text was found, or None
|
|
1166
|
+
|
|
1167
|
+
Examples:
|
|
1168
|
+
>>> location = bot.find_text("Confirmar", region=(0, 0, 800, 600), backtrack=True)
|
|
1169
|
+
>>> if location:
|
|
1170
|
+
... print(f"Text found at: {location}")
|
|
1171
|
+
"""
|
|
1172
|
+
import time
|
|
1173
|
+
|
|
1174
|
+
# If there are wait parameters, build a task and use the executor with wait
|
|
1175
|
+
if wait_until_found or wait_until_disappears:
|
|
1176
|
+
temp_task = {
|
|
1177
|
+
"text": text,
|
|
1178
|
+
"region": region,
|
|
1179
|
+
"char_type": filter_type,
|
|
1180
|
+
"confidence_threshold": confidence_threshold,
|
|
1181
|
+
"occurrence": occurrence,
|
|
1182
|
+
"wait_until_found": wait_until_found,
|
|
1183
|
+
"wait_until_disappears": wait_until_disappears,
|
|
1184
|
+
"wait_timeout": wait_timeout,
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
# Use the executor's wait-aware method
|
|
1188
|
+
if wait_until_found:
|
|
1189
|
+
return self.executor._find_text_with_wait(temp_task, 0)
|
|
1190
|
+
elif wait_until_disappears:
|
|
1191
|
+
return self.executor._find_text_with_wait(temp_task, 0)
|
|
1192
|
+
|
|
1193
|
+
# Original behavior without wait
|
|
1194
|
+
attempts = 0
|
|
1195
|
+
while attempts < max_attempts:
|
|
1196
|
+
if region is None:
|
|
1197
|
+
# If no region was specified, capture the whole screen
|
|
1198
|
+
import pyautogui
|
|
1199
|
+
|
|
1200
|
+
screen = pyautogui.screenshot()
|
|
1201
|
+
region = (0, 0, screen.width, screen.height)
|
|
1202
|
+
region_img = screen
|
|
1203
|
+
else:
|
|
1204
|
+
region_img = self.executor._capture_region(region)
|
|
1205
|
+
|
|
1206
|
+
# Whether the captured image already spans the whole region
|
|
1207
|
+
# (so box coords are already absolute) or needs offsetting.
|
|
1208
|
+
_full = region == (0, 0, region_img.width, region_img.height)
|
|
1209
|
+
_ox, _oy = (0, 0) if _full else (region[0], region[1])
|
|
1210
|
+
|
|
1211
|
+
# Backend-agnostic OCR narrative on the ``visus`` logger, so it
|
|
1212
|
+
# shows up in a session recording regardless of which OCR engine
|
|
1213
|
+
# is active (the engines' own loggers are noisy and filtered out).
|
|
1214
|
+
backend = self.config.get("ocr_backend", "rapidocr")
|
|
1215
|
+
logger.info(
|
|
1216
|
+
"OCR search: '%s' (occurrence=%d, filter=%s, backend=%s) in region %s",
|
|
1217
|
+
text, occurrence, filter_type, backend, region,
|
|
1218
|
+
)
|
|
1219
|
+
|
|
1220
|
+
if occurrence > 1 or word_box:
|
|
1221
|
+
# ``ocr_engine.find_text`` early-exits with only the first
|
|
1222
|
+
# high-confidence match (and only at line granularity), so it
|
|
1223
|
+
# can't serve the Nth occurrence nor a word-level box. Enumerate
|
|
1224
|
+
# ALL matches (deduped, reading order) and pick the Nth; pass
|
|
1225
|
+
# ``word_box`` so the box is tightened to the word itself.
|
|
1226
|
+
matches = self.ocr_engine.find_all_text(
|
|
1227
|
+
region_img, text, filter_type, confidence_threshold,
|
|
1228
|
+
word_box=word_box,
|
|
1229
|
+
)
|
|
1230
|
+
logger.info(
|
|
1231
|
+
"OCR found %d occurrence(s) of '%s'", len(matches), text
|
|
1232
|
+
)
|
|
1233
|
+
if len(matches) >= occurrence:
|
|
1234
|
+
conf, bx = matches[occurrence - 1]
|
|
1235
|
+
box = (_ox + bx[0], _oy + bx[1], bx[2], bx[3])
|
|
1236
|
+
logger.info(
|
|
1237
|
+
"Selected occurrence %d of '%s' at %s (confidence %.1f%%)",
|
|
1238
|
+
occurrence, text, box, conf,
|
|
1239
|
+
)
|
|
1240
|
+
return box
|
|
1241
|
+
logger.warning(
|
|
1242
|
+
"Occurrence %d of '%s' not found (%d match(es))",
|
|
1243
|
+
occurrence, text, len(matches),
|
|
1244
|
+
)
|
|
1245
|
+
else:
|
|
1246
|
+
found_boxes, scores, _ = self.ocr_engine.find_text(
|
|
1247
|
+
region_img, text, filter_type, confidence_threshold
|
|
1248
|
+
)
|
|
1249
|
+
if found_boxes:
|
|
1250
|
+
bx = found_boxes[0]
|
|
1251
|
+
box = (_ox + bx[0], _oy + bx[1], bx[2], bx[3])
|
|
1252
|
+
conf = scores[0] if scores else 0.0
|
|
1253
|
+
logger.info(
|
|
1254
|
+
"Found '%s' at %s (confidence %.1f%%)", text, box, conf
|
|
1255
|
+
)
|
|
1256
|
+
return box
|
|
1257
|
+
logger.warning("Text '%s' not found in region %s", text, region)
|
|
1258
|
+
|
|
1259
|
+
# If nothing was found and backtrack is enabled, try again
|
|
1260
|
+
if backtrack and attempts < max_attempts - 1:
|
|
1261
|
+
attempts += 1
|
|
1262
|
+
logger.info(f"Attempt {attempts}/{max_attempts} to find '{text}'")
|
|
1263
|
+
|
|
1264
|
+
# Adjust parameters for the next attempt
|
|
1265
|
+
confidence_threshold = max(60.0, confidence_threshold - 5.0)
|
|
1266
|
+
time.sleep(0.5)
|
|
1267
|
+
else:
|
|
1268
|
+
break
|
|
1269
|
+
|
|
1270
|
+
return None
|
|
1271
|
+
|
|
1272
|
+
def find_all_text(
|
|
1273
|
+
self,
|
|
1274
|
+
text,
|
|
1275
|
+
region=None,
|
|
1276
|
+
filter_type="both",
|
|
1277
|
+
confidence_threshold=None,
|
|
1278
|
+
word_box=False,
|
|
1279
|
+
):
|
|
1280
|
+
"""List every occurrence of ``text`` visible on screen (or in ``region``).
|
|
1281
|
+
|
|
1282
|
+
Unlike :meth:`find_text` (which stops at the first high-confidence
|
|
1283
|
+
hit), this scans the OCR output exhaustively and returns every
|
|
1284
|
+
distinct match. Use it when the same string appears multiple
|
|
1285
|
+
times and you need to disambiguate by position -- e.g. clicking
|
|
1286
|
+
the second ``"OK"`` on a form, or numbering visible rows.
|
|
1287
|
+
|
|
1288
|
+
Args:
|
|
1289
|
+
text (str): Substring to look for (case-insensitive, after
|
|
1290
|
+
``filter_type`` normalization).
|
|
1291
|
+
region (tuple, optional): ``(x, y, w, h)`` search area in
|
|
1292
|
+
screen coordinates. ``None`` searches the full screen.
|
|
1293
|
+
filter_type (str): ``"letters"`` / ``"numbers"`` / ``"both"``.
|
|
1294
|
+
confidence_threshold (float, optional): Minimum OCR confidence
|
|
1295
|
+
(0-100). Defaults to ``config.confidence_threshold``.
|
|
1296
|
+
word_box (bool): When True, narrow each box from the whole OCR
|
|
1297
|
+
text line to just ``text`` itself (handy to locate a word
|
|
1298
|
+
inside a long run such as a URL). Default False keeps the full
|
|
1299
|
+
line box. On RapidOCR this uses the engine's native per-word
|
|
1300
|
+
boxes (``return_word_box``), falling back to a character-offset
|
|
1301
|
+
estimate if unavailable; a no-op on Tesseract, whose boxes are
|
|
1302
|
+
already word-level.
|
|
1303
|
+
|
|
1304
|
+
Returns:
|
|
1305
|
+
list[tuple]: ``[(x, y, w, h, confidence), ...]`` in screen
|
|
1306
|
+
coordinates, sorted in reading order (top-to-bottom,
|
|
1307
|
+
left-to-right). Spatially overlapping detections collapse
|
|
1308
|
+
to a single entry that keeps the best confidence.
|
|
1309
|
+
|
|
1310
|
+
Examples:
|
|
1311
|
+
>>> matches = bot.find_all_text("Confirm")
|
|
1312
|
+
>>> for i, (x, y, w, h, conf) in enumerate(matches, start=1):
|
|
1313
|
+
... print(f" #{i} at ({x}, {y}) confidence={conf:.1f}%")
|
|
1314
|
+
"""
|
|
1315
|
+
import pyautogui
|
|
1316
|
+
|
|
1317
|
+
if region is None:
|
|
1318
|
+
screen = pyautogui.screenshot()
|
|
1319
|
+
region = (0, 0, screen.width, screen.height)
|
|
1320
|
+
region_img = screen
|
|
1321
|
+
else:
|
|
1322
|
+
region_img = self.executor._capture_region(region)
|
|
1323
|
+
|
|
1324
|
+
threshold = (
|
|
1325
|
+
confidence_threshold
|
|
1326
|
+
if confidence_threshold is not None
|
|
1327
|
+
else self.config.confidence_threshold
|
|
1328
|
+
)
|
|
1329
|
+
matches = self.ocr_engine.find_all_text(
|
|
1330
|
+
region_img, text, filter_type, threshold, word_box=word_box
|
|
1331
|
+
)
|
|
1332
|
+
# Box is relative to the captured region; convert to screen coordinates.
|
|
1333
|
+
return [
|
|
1334
|
+
(region[0] + box[0], region[1] + box[1], box[2], box[3], conf)
|
|
1335
|
+
for conf, box in matches
|
|
1336
|
+
]
|
|
1337
|
+
|
|
1338
|
+
def show_text_occurrences(
|
|
1339
|
+
self,
|
|
1340
|
+
text,
|
|
1341
|
+
region=None,
|
|
1342
|
+
filter_type="both",
|
|
1343
|
+
confidence_threshold=None,
|
|
1344
|
+
color="red",
|
|
1345
|
+
duration_ms=3000,
|
|
1346
|
+
width=3,
|
|
1347
|
+
word_box=False,
|
|
1348
|
+
):
|
|
1349
|
+
"""Overlay every occurrence of ``text`` on screen, numbered ``1, 2, 3...``.
|
|
1350
|
+
|
|
1351
|
+
Calls :meth:`find_all_text` and draws a click-through Win32
|
|
1352
|
+
overlay around each match with the position number above the
|
|
1353
|
+
top-left corner. Overlays fade after ``duration_ms`` and do not
|
|
1354
|
+
intercept clicks. On non-Windows the overlays are skipped but
|
|
1355
|
+
the match list is still returned.
|
|
1356
|
+
|
|
1357
|
+
Args:
|
|
1358
|
+
text (str): Substring to look for.
|
|
1359
|
+
region (tuple, optional): Search region.
|
|
1360
|
+
filter_type (str): OCR char filter.
|
|
1361
|
+
confidence_threshold (float, optional): Minimum confidence.
|
|
1362
|
+
color (str): Overlay color.
|
|
1363
|
+
duration_ms (int): How long each overlay stays on screen.
|
|
1364
|
+
width (int): Outline thickness in pixels.
|
|
1365
|
+
word_box (bool): Narrow each box to just ``text`` instead of the
|
|
1366
|
+
whole OCR line (see :meth:`find_all_text`). Default False.
|
|
1367
|
+
|
|
1368
|
+
Returns:
|
|
1369
|
+
list[tuple]: Same shape as :meth:`find_all_text`.
|
|
1370
|
+
|
|
1371
|
+
Examples:
|
|
1372
|
+
>>> matches = bot.show_text_occurrences("Login", duration_ms=4000)
|
|
1373
|
+
>>> print(f"Found {len(matches)} occurrence(s)")
|
|
1374
|
+
"""
|
|
1375
|
+
from .core.overlay_win32 import show_click_through_overlay
|
|
1376
|
+
|
|
1377
|
+
matches = self.find_all_text(
|
|
1378
|
+
text, region, filter_type, confidence_threshold, word_box=word_box
|
|
1379
|
+
)
|
|
1380
|
+
for i, (x, y, w, h, _) in enumerate(matches, start=1):
|
|
1381
|
+
show_click_through_overlay(
|
|
1382
|
+
(x, y, w, h),
|
|
1383
|
+
color=color,
|
|
1384
|
+
width=width,
|
|
1385
|
+
duration_ms=duration_ms,
|
|
1386
|
+
label=str(i),
|
|
1387
|
+
)
|
|
1388
|
+
logger.info(
|
|
1389
|
+
f"show_text_occurrences('{text}'): {len(matches)} match(es)"
|
|
1390
|
+
)
|
|
1391
|
+
return matches
|
|
1392
|
+
|
|
1393
|
+
@_records_in_session("text")
|
|
1394
|
+
def click_text(
|
|
1395
|
+
self,
|
|
1396
|
+
text,
|
|
1397
|
+
region=None,
|
|
1398
|
+
filter_type="both",
|
|
1399
|
+
delay=0,
|
|
1400
|
+
mouse_button="left",
|
|
1401
|
+
occurrence=1,
|
|
1402
|
+
backtrack=False,
|
|
1403
|
+
max_attempts=3,
|
|
1404
|
+
sendtext=None,
|
|
1405
|
+
confidence_threshold=None,
|
|
1406
|
+
show_overlay=None,
|
|
1407
|
+
wait_until_found=False,
|
|
1408
|
+
wait_until_disappears=False,
|
|
1409
|
+
wait_timeout=None,
|
|
1410
|
+
word_box=False,
|
|
1411
|
+
):
|
|
1412
|
+
"""
|
|
1413
|
+
Find and click on text with advanced features.
|
|
1414
|
+
|
|
1415
|
+
Args:
|
|
1416
|
+
text (str): Text to click on
|
|
1417
|
+
region (tuple, optional): Search region (x, y, width, height)
|
|
1418
|
+
filter_type (str): Filter type ("numbers", "letters", "both")
|
|
1419
|
+
delay (float): Delay after the click in seconds
|
|
1420
|
+
mouse_button (str): Mouse click type:
|
|
1421
|
+
• "left" - Simple left click (default)
|
|
1422
|
+
• "right" - Right click
|
|
1423
|
+
• "double" or "double left" - Double left click
|
|
1424
|
+
• "move_to" - Just moves the mouse without clicking
|
|
1425
|
+
occurrence (int): Which occurrence to click (1=first)
|
|
1426
|
+
backtrack (bool): Whether to use real backtracking between individual methods
|
|
1427
|
+
max_attempts (int): Maximum number of attempts
|
|
1428
|
+
sendtext (str, optional): Text to type after the click
|
|
1429
|
+
confidence_threshold (float, optional): Custom confidence threshold
|
|
1430
|
+
show_overlay (bool, optional): Whether to show the red overlay before the click
|
|
1431
|
+
If None, uses the global configuration
|
|
1432
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
|
|
1433
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
|
|
1434
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
1435
|
+
|
|
1436
|
+
Returns:
|
|
1437
|
+
bool: True if found and clicked, False otherwise
|
|
1438
|
+
"""
|
|
1439
|
+
# Use the global configuration if not specified
|
|
1440
|
+
if show_overlay is None:
|
|
1441
|
+
show_overlay = self.config.show_overlay
|
|
1442
|
+
|
|
1443
|
+
# If backtrack is True, use the individual backtrack system
|
|
1444
|
+
if backtrack:
|
|
1445
|
+
return self._execute_with_individual_backtrack(
|
|
1446
|
+
"click_text",
|
|
1447
|
+
self._click_text_internal,
|
|
1448
|
+
text,
|
|
1449
|
+
region,
|
|
1450
|
+
filter_type,
|
|
1451
|
+
delay,
|
|
1452
|
+
mouse_button,
|
|
1453
|
+
occurrence,
|
|
1454
|
+
max_attempts,
|
|
1455
|
+
sendtext,
|
|
1456
|
+
confidence_threshold,
|
|
1457
|
+
show_overlay,
|
|
1458
|
+
wait_until_found,
|
|
1459
|
+
wait_until_disappears,
|
|
1460
|
+
wait_timeout,
|
|
1461
|
+
word_box,
|
|
1462
|
+
backtrack=backtrack,
|
|
1463
|
+
)
|
|
1464
|
+
|
|
1465
|
+
# Original behavior without backtrack
|
|
1466
|
+
return self._click_text_internal(
|
|
1467
|
+
text,
|
|
1468
|
+
region,
|
|
1469
|
+
filter_type,
|
|
1470
|
+
delay,
|
|
1471
|
+
mouse_button,
|
|
1472
|
+
occurrence,
|
|
1473
|
+
max_attempts,
|
|
1474
|
+
sendtext,
|
|
1475
|
+
confidence_threshold,
|
|
1476
|
+
show_overlay,
|
|
1477
|
+
wait_until_found,
|
|
1478
|
+
wait_until_disappears,
|
|
1479
|
+
wait_timeout,
|
|
1480
|
+
word_box,
|
|
1481
|
+
)
|
|
1482
|
+
|
|
1483
|
+
def _click_text_internal(
|
|
1484
|
+
self,
|
|
1485
|
+
text,
|
|
1486
|
+
region=None,
|
|
1487
|
+
filter_type="both",
|
|
1488
|
+
delay=0,
|
|
1489
|
+
mouse_button="left",
|
|
1490
|
+
occurrence=1,
|
|
1491
|
+
max_attempts=3,
|
|
1492
|
+
sendtext=None,
|
|
1493
|
+
confidence_threshold=None,
|
|
1494
|
+
show_overlay=None,
|
|
1495
|
+
wait_until_found=False,
|
|
1496
|
+
wait_until_disappears=False,
|
|
1497
|
+
wait_timeout=None,
|
|
1498
|
+
word_box=False,
|
|
1499
|
+
):
|
|
1500
|
+
"""Internal version of click_text without backtrack (for use by the backtrack system)."""
|
|
1501
|
+
# Use the global configuration if not specified
|
|
1502
|
+
if show_overlay is None:
|
|
1503
|
+
show_overlay = self.config.show_overlay
|
|
1504
|
+
attempts = 0
|
|
1505
|
+
while attempts < max_attempts:
|
|
1506
|
+
attempts += 1
|
|
1507
|
+
|
|
1508
|
+
try:
|
|
1509
|
+
if attempts > 1:
|
|
1510
|
+
logger.info(
|
|
1511
|
+
f"Attempt {attempts}/{max_attempts} to click on '{text}'"
|
|
1512
|
+
)
|
|
1513
|
+
|
|
1514
|
+
# Use the custom threshold if provided, otherwise use the one from the configuration
|
|
1515
|
+
threshold = confidence_threshold or self.config.confidence_threshold
|
|
1516
|
+
|
|
1517
|
+
location = self.find_text(
|
|
1518
|
+
text,
|
|
1519
|
+
region,
|
|
1520
|
+
filter_type,
|
|
1521
|
+
threshold,
|
|
1522
|
+
occurrence,
|
|
1523
|
+
max_attempts=1,
|
|
1524
|
+
backtrack=False,
|
|
1525
|
+
wait_until_found=wait_until_found,
|
|
1526
|
+
wait_until_disappears=wait_until_disappears,
|
|
1527
|
+
wait_timeout=wait_timeout,
|
|
1528
|
+
word_box=word_box,
|
|
1529
|
+
)
|
|
1530
|
+
|
|
1531
|
+
if location:
|
|
1532
|
+
# Build a temporary task to use the executor
|
|
1533
|
+
temp_task = {
|
|
1534
|
+
"mouse_button": mouse_button,
|
|
1535
|
+
"delay": delay,
|
|
1536
|
+
"sendtext": sendtext,
|
|
1537
|
+
"show_overlay": show_overlay,
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
try:
|
|
1541
|
+
self.executor._perform_action(temp_task, location)
|
|
1542
|
+
return True
|
|
1543
|
+
except Exception as e:
|
|
1544
|
+
logger.error(f"Error clicking on text: {e}")
|
|
1545
|
+
if attempts >= max_attempts:
|
|
1546
|
+
return False
|
|
1547
|
+
time.sleep(0.5)
|
|
1548
|
+
else:
|
|
1549
|
+
logger.warning(f"Text '{text}' not found on attempt {attempts}")
|
|
1550
|
+
if attempts < max_attempts:
|
|
1551
|
+
time.sleep(0.5)
|
|
1552
|
+
|
|
1553
|
+
except Exception as e:
|
|
1554
|
+
logger.error(f"Error on attempt {attempts} for '{text}': {e}")
|
|
1555
|
+
if attempts >= max_attempts:
|
|
1556
|
+
return False
|
|
1557
|
+
time.sleep(0.5)
|
|
1558
|
+
|
|
1559
|
+
return False
|
|
1560
|
+
|
|
1561
|
+
def _click_image_internal(
|
|
1562
|
+
self,
|
|
1563
|
+
image_path,
|
|
1564
|
+
region=None,
|
|
1565
|
+
confidence=0.9,
|
|
1566
|
+
delay=0,
|
|
1567
|
+
mouse_button="left",
|
|
1568
|
+
max_attempts=3,
|
|
1569
|
+
specific=True,
|
|
1570
|
+
sendtext=None,
|
|
1571
|
+
show_overlay=None,
|
|
1572
|
+
wait_until_found=False,
|
|
1573
|
+
wait_until_disappears=False,
|
|
1574
|
+
wait_timeout=None,
|
|
1575
|
+
occurrence=1,
|
|
1576
|
+
):
|
|
1577
|
+
"""Internal version of click_image without backtrack (for use by the backtrack system)."""
|
|
1578
|
+
# Use the global configuration if not specified
|
|
1579
|
+
if show_overlay is None:
|
|
1580
|
+
show_overlay = self.config.show_overlay
|
|
1581
|
+
|
|
1582
|
+
# For click_image, wait_until_disappears must wait AFTER the click, not before
|
|
1583
|
+
# So we pass only wait_until_found to find_image
|
|
1584
|
+
location = self.find_image(
|
|
1585
|
+
image_path,
|
|
1586
|
+
region,
|
|
1587
|
+
confidence,
|
|
1588
|
+
max_attempts,
|
|
1589
|
+
False,
|
|
1590
|
+
specific,
|
|
1591
|
+
wait_until_found,
|
|
1592
|
+
False,
|
|
1593
|
+
wait_timeout,
|
|
1594
|
+
occurrence=occurrence,
|
|
1595
|
+
) # backtrack=False for find_image
|
|
1596
|
+
|
|
1597
|
+
if location:
|
|
1598
|
+
# Build a temporary task to use the executor
|
|
1599
|
+
temp_task = {
|
|
1600
|
+
"image": image_path,
|
|
1601
|
+
"region": region,
|
|
1602
|
+
"confidence": confidence,
|
|
1603
|
+
"specific": specific,
|
|
1604
|
+
"mouse_button": mouse_button,
|
|
1605
|
+
"delay": delay,
|
|
1606
|
+
"sendtext": sendtext,
|
|
1607
|
+
"show_overlay": show_overlay,
|
|
1608
|
+
"wait_until_disappears": wait_until_disappears,
|
|
1609
|
+
"wait_timeout": wait_timeout,
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
try:
|
|
1613
|
+
self.executor._perform_action(temp_task, location)
|
|
1614
|
+
return True
|
|
1615
|
+
except Exception as e:
|
|
1616
|
+
logger.error(f"Error clicking on image: {e}")
|
|
1617
|
+
return False
|
|
1618
|
+
|
|
1619
|
+
return False
|
|
1620
|
+
|
|
1621
|
+
def extract_text_from_region(
|
|
1622
|
+
self,
|
|
1623
|
+
region=None,
|
|
1624
|
+
filter_type="both",
|
|
1625
|
+
confidence_threshold=50.0,
|
|
1626
|
+
return_full_data=False,
|
|
1627
|
+
max_attempts=3,
|
|
1628
|
+
backtrack=False,
|
|
1629
|
+
):
|
|
1630
|
+
"""
|
|
1631
|
+
Extract all text found in a specific region of the screen.
|
|
1632
|
+
|
|
1633
|
+
Args:
|
|
1634
|
+
region (tuple, optional): (x, y, width, height) of the region to extract text from.
|
|
1635
|
+
If None, extracts from the whole screen
|
|
1636
|
+
filter_type (str): Filter type for the text:
|
|
1637
|
+
• "numbers" - Numbers only
|
|
1638
|
+
• "letters" - Letters only
|
|
1639
|
+
• "both" - Numbers and letters (default)
|
|
1640
|
+
confidence_threshold (float): Minimum confidence threshold (0-100)
|
|
1641
|
+
return_full_data (bool): If True, returns full data with coordinates and confidence
|
|
1642
|
+
max_attempts (int): Maximum number of attempts if backtrack=True
|
|
1643
|
+
backtrack (bool): Whether to try multiple times with adjustments
|
|
1644
|
+
|
|
1645
|
+
Returns:
|
|
1646
|
+
If return_full_data=False:
|
|
1647
|
+
list: List of strings with all the text found
|
|
1648
|
+
If return_full_data=True:
|
|
1649
|
+
list: List of dictionaries with:
|
|
1650
|
+
- 'text': Text found
|
|
1651
|
+
- 'confidence': Confidence level (0-100)
|
|
1652
|
+
- 'box': Coordinates (x, y, width, height)
|
|
1653
|
+
- 'absolute_box': Absolute coordinates on screen
|
|
1654
|
+
|
|
1655
|
+
Examples:
|
|
1656
|
+
Simple extraction:
|
|
1657
|
+
>>> texts = bot.extract_text_from_region(region=(100, 100, 400, 300))
|
|
1658
|
+
>>> print("Texts found:", texts)
|
|
1659
|
+
|
|
1660
|
+
Extraction with full data:
|
|
1661
|
+
>>> data = bot.extract_text_from_region(
|
|
1662
|
+
... region=(100, 100, 400, 300),
|
|
1663
|
+
... return_full_data=True,
|
|
1664
|
+
... confidence_threshold=70.0
|
|
1665
|
+
... )
|
|
1666
|
+
>>> for item in data:
|
|
1667
|
+
... print(f"'{item['text']}' - Confidence: {item['confidence']:.1f}% - Position: {item['absolute_box']}")
|
|
1668
|
+
|
|
1669
|
+
Numbers only:
|
|
1670
|
+
>>> numbers = bot.extract_text_from_region(
|
|
1671
|
+
... region=(50, 50, 200, 100),
|
|
1672
|
+
... filter_type="numbers"
|
|
1673
|
+
... )
|
|
1674
|
+
"""
|
|
1675
|
+
import time
|
|
1676
|
+
|
|
1677
|
+
attempts = 0
|
|
1678
|
+
while attempts < max_attempts:
|
|
1679
|
+
try:
|
|
1680
|
+
if region is None:
|
|
1681
|
+
# If no region was specified, capture the whole screen
|
|
1682
|
+
import pyautogui
|
|
1683
|
+
|
|
1684
|
+
screen = pyautogui.screenshot()
|
|
1685
|
+
region = (0, 0, screen.width, screen.height)
|
|
1686
|
+
region_img = screen
|
|
1687
|
+
else:
|
|
1688
|
+
region_img = self.executor._capture_region(region)
|
|
1689
|
+
|
|
1690
|
+
# Use the OCR engine's extract_all_text method
|
|
1691
|
+
ocr_results = self.ocr_engine.extract_all_text(region_img, filter_type)
|
|
1692
|
+
|
|
1693
|
+
# Filter by confidence
|
|
1694
|
+
filtered_results = [
|
|
1695
|
+
result
|
|
1696
|
+
for result in ocr_results
|
|
1697
|
+
if result.confidence >= confidence_threshold
|
|
1698
|
+
]
|
|
1699
|
+
|
|
1700
|
+
if not filtered_results and backtrack and attempts < max_attempts - 1:
|
|
1701
|
+
# If nothing was found and backtrack is enabled, adjust parameters
|
|
1702
|
+
attempts += 1
|
|
1703
|
+
logger.info(
|
|
1704
|
+
f"Attempt {attempts}/{max_attempts} for text extraction"
|
|
1705
|
+
)
|
|
1706
|
+
confidence_threshold = max(30.0, confidence_threshold - 10.0)
|
|
1707
|
+
time.sleep(0.5)
|
|
1708
|
+
continue
|
|
1709
|
+
|
|
1710
|
+
if return_full_data:
|
|
1711
|
+
# Return full data with absolute coordinates
|
|
1712
|
+
full_data = []
|
|
1713
|
+
for result in filtered_results:
|
|
1714
|
+
absolute_box = (
|
|
1715
|
+
(
|
|
1716
|
+
region[0] + result.box[0],
|
|
1717
|
+
region[1] + result.box[1],
|
|
1718
|
+
result.box[2],
|
|
1719
|
+
result.box[3],
|
|
1720
|
+
)
|
|
1721
|
+
if region != (0, 0, region_img.width, region_img.height)
|
|
1722
|
+
else result.box
|
|
1723
|
+
)
|
|
1724
|
+
|
|
1725
|
+
full_data.append(
|
|
1726
|
+
{
|
|
1727
|
+
"text": result.text,
|
|
1728
|
+
"confidence": result.confidence,
|
|
1729
|
+
"box": result.box, # Coordinates relative to the region
|
|
1730
|
+
"absolute_box": absolute_box, # Absolute coordinates on screen
|
|
1731
|
+
}
|
|
1732
|
+
)
|
|
1733
|
+
|
|
1734
|
+
return full_data
|
|
1735
|
+
else:
|
|
1736
|
+
# Return only the texts
|
|
1737
|
+
return [result.text for result in filtered_results]
|
|
1738
|
+
|
|
1739
|
+
except Exception as e:
|
|
1740
|
+
logger.error(f"Error on attempt {attempts} for text extraction: {e}")
|
|
1741
|
+
if backtrack and attempts < max_attempts - 1:
|
|
1742
|
+
attempts += 1
|
|
1743
|
+
time.sleep(0.5)
|
|
1744
|
+
else:
|
|
1745
|
+
break
|
|
1746
|
+
|
|
1747
|
+
# If it got here, it failed
|
|
1748
|
+
return [] if not return_full_data else []
|
|
1749
|
+
|
|
1750
|
+
def get_last_extracted_text(self):
|
|
1751
|
+
"""
|
|
1752
|
+
Get the last extracted text when used in task lists.
|
|
1753
|
+
|
|
1754
|
+
Returns:
|
|
1755
|
+
list: Last text extraction result, or None if no extraction has been done
|
|
1756
|
+
|
|
1757
|
+
Examples:
|
|
1758
|
+
>>> tasks = [
|
|
1759
|
+
... {
|
|
1760
|
+
... 'type': 'extract_text',
|
|
1761
|
+
... 'region': (100, 100, 400, 300),
|
|
1762
|
+
... 'filter_type': 'both'
|
|
1763
|
+
... }
|
|
1764
|
+
... ]
|
|
1765
|
+
>>> results = bot.execute_tasks(tasks)
|
|
1766
|
+
>>> extracted_texts = bot.get_last_extracted_text()
|
|
1767
|
+
>>> print("Texts found:", extracted_texts)
|
|
1768
|
+
"""
|
|
1769
|
+
return getattr(self.executor, "last_extracted_text", None)
|
|
1770
|
+
|
|
1771
|
+
def find_image(
|
|
1772
|
+
self,
|
|
1773
|
+
image_path,
|
|
1774
|
+
region=None,
|
|
1775
|
+
confidence=0.9,
|
|
1776
|
+
max_attempts=3,
|
|
1777
|
+
backtrack=False,
|
|
1778
|
+
specific=True,
|
|
1779
|
+
wait_until_found=False,
|
|
1780
|
+
wait_until_disappears=False,
|
|
1781
|
+
wait_timeout=None,
|
|
1782
|
+
scales=None,
|
|
1783
|
+
occurrence=1,
|
|
1784
|
+
):
|
|
1785
|
+
"""
|
|
1786
|
+
Find an image on screen with all the advanced features.
|
|
1787
|
+
|
|
1788
|
+
Args:
|
|
1789
|
+
image_path (str): Path to the image
|
|
1790
|
+
region (tuple, optional): Search region
|
|
1791
|
+
confidence (float): Confidence level
|
|
1792
|
+
max_attempts (int): Maximum number of attempts if backtrack=True
|
|
1793
|
+
backtrack (bool): Whether to try multiple times with adjustments
|
|
1794
|
+
specific (bool): If True, searches in the region; if False, searches the whole screen + scale variations
|
|
1795
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
1796
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
1797
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
1798
|
+
scales (list, optional): List of scales to try (e.g.: [1.0, 0.95, 1.05])
|
|
1799
|
+
occurrence (int): Which match to return when the template appears more
|
|
1800
|
+
than once, 1-based in reading order (top-to-bottom, left-to-right).
|
|
1801
|
+
``occurrence > 1`` enumerates every match and picks the Nth.
|
|
1802
|
+
|
|
1803
|
+
Returns:
|
|
1804
|
+
tuple: Coordinates of the image or None
|
|
1805
|
+
"""
|
|
1806
|
+
import time
|
|
1807
|
+
|
|
1808
|
+
image_path = self.image_map.resolve(image_path)
|
|
1809
|
+
|
|
1810
|
+
# occurrence > 1: a single-best match (the default find path) can't serve
|
|
1811
|
+
# the 2nd/3rd copy. Enumerate every match and pick the Nth in reading
|
|
1812
|
+
# order (top-to-bottom, left-to-right) -- mirrors click_text(occurrence=N).
|
|
1813
|
+
if occurrence and occurrence > 1:
|
|
1814
|
+
from .core.locate import find_all as _find_all
|
|
1815
|
+
|
|
1816
|
+
# Honor `specific` exactly like the single-match path below: restrict
|
|
1817
|
+
# to `region` only when specific=True. With specific=False (search the
|
|
1818
|
+
# whole screen) a captured template carries the FIRST copy's box as
|
|
1819
|
+
# `region`, so restricting find_all to it would only ever see 1 match
|
|
1820
|
+
# and the 2nd/3rd copies (outside that box) could never be selected.
|
|
1821
|
+
search_region = tuple(region) if (specific and region) else None
|
|
1822
|
+
boxes = _find_all(
|
|
1823
|
+
image_path,
|
|
1824
|
+
region=search_region,
|
|
1825
|
+
confidence=confidence,
|
|
1826
|
+
)
|
|
1827
|
+
boxes = sorted(boxes, key=lambda b: (b[1], b[0]))
|
|
1828
|
+
logger.info(
|
|
1829
|
+
"Image '%s' found %d occurrence(s)", image_path, len(boxes)
|
|
1830
|
+
)
|
|
1831
|
+
if len(boxes) >= occurrence:
|
|
1832
|
+
box = boxes[occurrence - 1]
|
|
1833
|
+
logger.info(
|
|
1834
|
+
"Selected image occurrence %d at %s", occurrence, box
|
|
1835
|
+
)
|
|
1836
|
+
return box
|
|
1837
|
+
logger.warning(
|
|
1838
|
+
"Image occurrence %d of '%s' not found (%d match(es))",
|
|
1839
|
+
occurrence, image_path, len(boxes),
|
|
1840
|
+
)
|
|
1841
|
+
return None
|
|
1842
|
+
|
|
1843
|
+
if scales is None:
|
|
1844
|
+
scales = [1.0, 0.95, 1.05] if not specific else [1.0]
|
|
1845
|
+
|
|
1846
|
+
# If there are wait parameters, build a task and use the executor with wait
|
|
1847
|
+
if wait_until_found or wait_until_disappears:
|
|
1848
|
+
temp_task = {
|
|
1849
|
+
"image": image_path,
|
|
1850
|
+
"region": region,
|
|
1851
|
+
"confidence": confidence,
|
|
1852
|
+
"specific": specific,
|
|
1853
|
+
"wait_until_found": wait_until_found,
|
|
1854
|
+
"wait_until_disappears": wait_until_disappears,
|
|
1855
|
+
"wait_timeout": wait_timeout,
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
# Use the executor's wait-aware method
|
|
1859
|
+
return self.executor._find_image_with_wait(temp_task, 0)
|
|
1860
|
+
|
|
1861
|
+
# Original behavior without wait
|
|
1862
|
+
attempts = 0
|
|
1863
|
+
while attempts < max_attempts:
|
|
1864
|
+
try:
|
|
1865
|
+
# Try with different scales if not specific
|
|
1866
|
+
for scale in scales:
|
|
1867
|
+
try:
|
|
1868
|
+
if scale != 1.0 and not specific:
|
|
1869
|
+
# Resize the reference image
|
|
1870
|
+
import os
|
|
1871
|
+
|
|
1872
|
+
from PIL import Image
|
|
1873
|
+
|
|
1874
|
+
ref_img = Image.open(image_path)
|
|
1875
|
+
new_width = int(ref_img.width * scale)
|
|
1876
|
+
new_height = int(ref_img.height * scale)
|
|
1877
|
+
scaled_img = ref_img.resize((new_width, new_height))
|
|
1878
|
+
|
|
1879
|
+
# Save temporarily
|
|
1880
|
+
temp_path = f"temp_scaled_{scale}_{attempts}.png"
|
|
1881
|
+
scaled_img.save(temp_path)
|
|
1882
|
+
|
|
1883
|
+
# NEW LOGIC: specific controls where to search
|
|
1884
|
+
if specific and region:
|
|
1885
|
+
# If specific AND there is a region, search in the region
|
|
1886
|
+
location = self.executor._locate_image_with_retry(
|
|
1887
|
+
temp_path, region, confidence
|
|
1888
|
+
)
|
|
1889
|
+
else:
|
|
1890
|
+
# If not specific OR no region, search the whole screen
|
|
1891
|
+
location = self.executor._locate_image_with_retry(
|
|
1892
|
+
temp_path, None, confidence
|
|
1893
|
+
)
|
|
1894
|
+
|
|
1895
|
+
# Remove the temporary file
|
|
1896
|
+
if os.path.exists(temp_path):
|
|
1897
|
+
os.remove(temp_path)
|
|
1898
|
+
else:
|
|
1899
|
+
# NEW LOGIC: specific controls where to search
|
|
1900
|
+
if specific and region:
|
|
1901
|
+
# If specific AND there is a region, search in the region
|
|
1902
|
+
logger.info(
|
|
1903
|
+
f"Searching for image in region {region} (specific=True)"
|
|
1904
|
+
)
|
|
1905
|
+
location = self.executor._locate_image_with_retry(
|
|
1906
|
+
image_path, region, confidence
|
|
1907
|
+
)
|
|
1908
|
+
else:
|
|
1909
|
+
# If not specific OR no region, search the whole screen
|
|
1910
|
+
logger.info(
|
|
1911
|
+
"Searching for image across the whole screen (specific=False)"
|
|
1912
|
+
)
|
|
1913
|
+
location = self.executor._locate_image_with_retry(
|
|
1914
|
+
image_path, None, confidence
|
|
1915
|
+
)
|
|
1916
|
+
|
|
1917
|
+
if location:
|
|
1918
|
+
return location
|
|
1919
|
+
|
|
1920
|
+
except Exception as e:
|
|
1921
|
+
logger.debug(f"Error searching for image at scale {scale}: {e}")
|
|
1922
|
+
continue
|
|
1923
|
+
# If nothing was found and backtrack is enabled, adjust parameters
|
|
1924
|
+
if backtrack and attempts < max_attempts - 1:
|
|
1925
|
+
attempts += 1
|
|
1926
|
+
logger.info(f"Attempt {attempts}/{max_attempts} to find the image")
|
|
1927
|
+
|
|
1928
|
+
# Gradually lower the confidence
|
|
1929
|
+
confidence = max(0.7, confidence - 0.05)
|
|
1930
|
+
time.sleep(0.5)
|
|
1931
|
+
else:
|
|
1932
|
+
break
|
|
1933
|
+
|
|
1934
|
+
except Exception as e:
|
|
1935
|
+
logger.error(f"Error searching for image: {e}")
|
|
1936
|
+
if not backtrack:
|
|
1937
|
+
break
|
|
1938
|
+
|
|
1939
|
+
return None
|
|
1940
|
+
|
|
1941
|
+
@_records_in_session("image")
|
|
1942
|
+
def click_image(
|
|
1943
|
+
self,
|
|
1944
|
+
image_path,
|
|
1945
|
+
region=None,
|
|
1946
|
+
confidence=0.9,
|
|
1947
|
+
delay=0,
|
|
1948
|
+
mouse_button="left",
|
|
1949
|
+
max_attempts=3,
|
|
1950
|
+
backtrack=False,
|
|
1951
|
+
specific=True,
|
|
1952
|
+
sendtext=None,
|
|
1953
|
+
show_overlay=None,
|
|
1954
|
+
wait_until_found=False,
|
|
1955
|
+
wait_until_disappears=False,
|
|
1956
|
+
wait_timeout=None,
|
|
1957
|
+
occurrence=1,
|
|
1958
|
+
):
|
|
1959
|
+
"""
|
|
1960
|
+
Find and click on an image with all the advanced features.
|
|
1961
|
+
|
|
1962
|
+
Args:
|
|
1963
|
+
image_path (str): Path to the image
|
|
1964
|
+
region (tuple, optional): Search region (x, y, width, height)
|
|
1965
|
+
confidence (float): Confidence level (0.0-1.0)
|
|
1966
|
+
delay (float): Delay after the click in seconds
|
|
1967
|
+
mouse_button (str): Mouse click type:
|
|
1968
|
+
• "left" - Simple left click (default)
|
|
1969
|
+
• "right" - Right click
|
|
1970
|
+
• "double" or "double left" - Double left click
|
|
1971
|
+
• "move_to" - Just moves the mouse without clicking
|
|
1972
|
+
max_attempts (int): Maximum number of attempts if backtrack=True
|
|
1973
|
+
backtrack (bool): Whether to use real backtracking between individual methods
|
|
1974
|
+
specific (bool): If True, exact search; if False, allows variations
|
|
1975
|
+
sendtext (str, optional): Text to type after the click
|
|
1976
|
+
show_overlay (bool, optional): Whether to show the red overlay before the click
|
|
1977
|
+
If None, uses the global configuration
|
|
1978
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
1979
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
1980
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
1981
|
+
|
|
1982
|
+
Returns:
|
|
1983
|
+
bool: True if found and clicked, False otherwise
|
|
1984
|
+
"""
|
|
1985
|
+
# Use the global configuration if not specified
|
|
1986
|
+
if show_overlay is None:
|
|
1987
|
+
show_overlay = self.config.show_overlay
|
|
1988
|
+
|
|
1989
|
+
image_path = self.image_map.resolve(image_path)
|
|
1990
|
+
|
|
1991
|
+
# If backtrack is True, use the individual backtrack system
|
|
1992
|
+
if backtrack:
|
|
1993
|
+
return self._execute_with_individual_backtrack(
|
|
1994
|
+
"click_image",
|
|
1995
|
+
self._click_image_internal,
|
|
1996
|
+
image_path,
|
|
1997
|
+
region,
|
|
1998
|
+
confidence,
|
|
1999
|
+
delay,
|
|
2000
|
+
mouse_button,
|
|
2001
|
+
max_attempts,
|
|
2002
|
+
specific,
|
|
2003
|
+
sendtext,
|
|
2004
|
+
show_overlay,
|
|
2005
|
+
wait_until_found,
|
|
2006
|
+
wait_until_disappears,
|
|
2007
|
+
wait_timeout,
|
|
2008
|
+
occurrence,
|
|
2009
|
+
backtrack=backtrack,
|
|
2010
|
+
)
|
|
2011
|
+
|
|
2012
|
+
# Original behavior without backtrack
|
|
2013
|
+
return self._click_image_internal(
|
|
2014
|
+
image_path,
|
|
2015
|
+
region,
|
|
2016
|
+
confidence,
|
|
2017
|
+
delay,
|
|
2018
|
+
mouse_button,
|
|
2019
|
+
max_attempts,
|
|
2020
|
+
specific,
|
|
2021
|
+
sendtext,
|
|
2022
|
+
show_overlay,
|
|
2023
|
+
wait_until_found,
|
|
2024
|
+
wait_until_disappears,
|
|
2025
|
+
wait_timeout,
|
|
2026
|
+
occurrence,
|
|
2027
|
+
)
|
|
2028
|
+
|
|
2029
|
+
@_records_in_session("click")
|
|
2030
|
+
def click_at(self, location, mouse_button="left", delay=0, show_overlay=None):
|
|
2031
|
+
"""
|
|
2032
|
+
Click on specific screen coordinates.
|
|
2033
|
+
|
|
2034
|
+
Args:
|
|
2035
|
+
location (tuple): (x, y, width, height) of the location
|
|
2036
|
+
mouse_button (str): Mouse button ("left", "right", "double", "move_to")
|
|
2037
|
+
delay (float): Delay after the click
|
|
2038
|
+
show_overlay (bool, optional): Whether to show the red overlay before the click
|
|
2039
|
+
If None, uses the global configuration
|
|
2040
|
+
|
|
2041
|
+
Returns:
|
|
2042
|
+
bool: True if clicked successfully
|
|
2043
|
+
|
|
2044
|
+
Examples:
|
|
2045
|
+
>>> bot.click_at((100, 200, 50, 30), mouse_button='left', delay=1)
|
|
2046
|
+
"""
|
|
2047
|
+
try:
|
|
2048
|
+
# Use the global configuration if not specified
|
|
2049
|
+
if show_overlay is None:
|
|
2050
|
+
show_overlay = self.config.show_overlay
|
|
2051
|
+
|
|
2052
|
+
# Build a temporary task to use the executor
|
|
2053
|
+
temp_task = {
|
|
2054
|
+
"mouse_button": mouse_button,
|
|
2055
|
+
"delay": delay,
|
|
2056
|
+
"show_overlay": show_overlay,
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
self.executor._perform_action(temp_task, location)
|
|
2060
|
+
return True
|
|
2061
|
+
|
|
2062
|
+
except Exception as e:
|
|
2063
|
+
logger.error(f"Error clicking on coordinates {location}: {e}")
|
|
2064
|
+
return False
|
|
2065
|
+
|
|
2066
|
+
def find_relative_image(
|
|
2067
|
+
self,
|
|
2068
|
+
anchor_image,
|
|
2069
|
+
target_image,
|
|
2070
|
+
max_distance=200,
|
|
2071
|
+
confidence=0.9,
|
|
2072
|
+
target_region=None,
|
|
2073
|
+
wait_until_found=False,
|
|
2074
|
+
wait_until_disappears=False,
|
|
2075
|
+
wait_timeout=None,
|
|
2076
|
+
):
|
|
2077
|
+
"""
|
|
2078
|
+
Find a target image near an anchor image.
|
|
2079
|
+
|
|
2080
|
+
Args:
|
|
2081
|
+
anchor_image (str): Path to the anchor image (unique on screen)
|
|
2082
|
+
target_image (str): Path to the target image (there may be multiple)
|
|
2083
|
+
max_distance (int): Maximum distance in pixels from the anchor to the target
|
|
2084
|
+
confidence (float): Confidence level for detection (0.0-1.0)
|
|
2085
|
+
target_region (tuple, optional): Specific region to search for the target (x, y, width, height)
|
|
2086
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
2087
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
2088
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
2089
|
+
|
|
2090
|
+
Returns:
|
|
2091
|
+
tuple: Location of the target image closest to the anchor, or None
|
|
2092
|
+
|
|
2093
|
+
Examples:
|
|
2094
|
+
>>> location = bot.find_relative_image('anchor.png', 'target.png', max_distance=150)
|
|
2095
|
+
>>> if location:
|
|
2096
|
+
... print(f"Target found near the anchor: {location}")
|
|
2097
|
+
"""
|
|
2098
|
+
# If there are wait parameters, build a task and use the executor with wait
|
|
2099
|
+
if wait_until_found or wait_until_disappears:
|
|
2100
|
+
# Build a search function for the wait manager
|
|
2101
|
+
def search_relative():
|
|
2102
|
+
try:
|
|
2103
|
+
return self.executor.relative_detector.locate_relative_image(
|
|
2104
|
+
anchor_image,
|
|
2105
|
+
target_image,
|
|
2106
|
+
confidence,
|
|
2107
|
+
max_distance,
|
|
2108
|
+
target_region,
|
|
2109
|
+
)
|
|
2110
|
+
except Exception:
|
|
2111
|
+
return None
|
|
2112
|
+
|
|
2113
|
+
if wait_until_found:
|
|
2114
|
+
return self.executor.wait_manager.wait_until_found(
|
|
2115
|
+
search_relative,
|
|
2116
|
+
timeout=wait_timeout or self.config.get("wait_timeout", 30),
|
|
2117
|
+
description=f"relative image '{target_image}' near '{anchor_image}'",
|
|
2118
|
+
)
|
|
2119
|
+
elif wait_until_disappears:
|
|
2120
|
+
# First find the image
|
|
2121
|
+
result = search_relative()
|
|
2122
|
+
if not result:
|
|
2123
|
+
return None
|
|
2124
|
+
|
|
2125
|
+
# Then wait for it to disappear
|
|
2126
|
+
disappeared = self.executor.wait_manager.wait_until_disappears(
|
|
2127
|
+
search_relative,
|
|
2128
|
+
timeout=wait_timeout or self.config.get("wait_timeout", 30),
|
|
2129
|
+
description=f"relative image '{target_image}' near '{anchor_image}'",
|
|
2130
|
+
)
|
|
2131
|
+
return result if disappeared else None
|
|
2132
|
+
|
|
2133
|
+
# Original behavior without wait
|
|
2134
|
+
try:
|
|
2135
|
+
return self.executor.relative_detector.locate_relative_image(
|
|
2136
|
+
anchor_image, target_image, confidence, max_distance, target_region
|
|
2137
|
+
)
|
|
2138
|
+
except Exception as e:
|
|
2139
|
+
logger.error(f"Error during relative image search: {e}")
|
|
2140
|
+
return None
|
|
2141
|
+
|
|
2142
|
+
@_records_in_session("relative_image")
|
|
2143
|
+
def click_relative_image(
|
|
2144
|
+
self,
|
|
2145
|
+
anchor_image,
|
|
2146
|
+
target_image,
|
|
2147
|
+
max_distance=200,
|
|
2148
|
+
confidence=0.9,
|
|
2149
|
+
target_region=None,
|
|
2150
|
+
delay=0,
|
|
2151
|
+
mouse_button="left",
|
|
2152
|
+
backtrack=False,
|
|
2153
|
+
max_attempts=3,
|
|
2154
|
+
sendtext=None,
|
|
2155
|
+
wait_until_found=False,
|
|
2156
|
+
wait_until_disappears=False,
|
|
2157
|
+
wait_timeout=None,
|
|
2158
|
+
):
|
|
2159
|
+
"""
|
|
2160
|
+
Click on a target image near an anchor image.
|
|
2161
|
+
|
|
2162
|
+
Args:
|
|
2163
|
+
anchor_image (str): Path to the anchor image
|
|
2164
|
+
target_image (str): Path to the target image
|
|
2165
|
+
max_distance (int): Maximum distance in pixels
|
|
2166
|
+
confidence (float): Confidence level (0.0-1.0)
|
|
2167
|
+
target_region (tuple, optional): Region to search for the target (x, y, width, height)
|
|
2168
|
+
delay (float): Delay after the click in seconds
|
|
2169
|
+
mouse_button (str): Mouse click type:
|
|
2170
|
+
• "left" - Simple left click (default)
|
|
2171
|
+
• "right" - Right click
|
|
2172
|
+
• "double" or "double left" - Double left click
|
|
2173
|
+
• "move_to" - Just moves the mouse without clicking
|
|
2174
|
+
backtrack (bool): Whether to use backtracking on failure
|
|
2175
|
+
max_attempts (int): Maximum number of attempts
|
|
2176
|
+
sendtext (str, optional): Text to type after the click
|
|
2177
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
2178
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
2179
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
2180
|
+
|
|
2181
|
+
Returns:
|
|
2182
|
+
bool: True if found and clicked, False otherwise
|
|
2183
|
+
|
|
2184
|
+
Examples:
|
|
2185
|
+
>>> # Simple left click (default)
|
|
2186
|
+
>>> success = bot.click_relative_image('anchor.png', 'target.png',
|
|
2187
|
+
... max_distance=150, mouse_button="left")
|
|
2188
|
+
|
|
2189
|
+
>>> # Right click
|
|
2190
|
+
>>> success = bot.click_relative_image('anchor.png', 'target.png',
|
|
2191
|
+
... mouse_button="right", backtrack=True)
|
|
2192
|
+
|
|
2193
|
+
>>> # Double left click
|
|
2194
|
+
>>> success = bot.click_relative_image('anchor.png', 'target.png',
|
|
2195
|
+
... mouse_button="double", delay=1.0)
|
|
2196
|
+
|
|
2197
|
+
>>> # Just move the mouse to the position (without clicking)
|
|
2198
|
+
>>> success = bot.click_relative_image('anchor.png', 'target.png',
|
|
2199
|
+
... mouse_button="move_to", delay=0.5)
|
|
2200
|
+
|
|
2201
|
+
>>> # Click and type text afterwards
|
|
2202
|
+
>>> success = bot.click_relative_image('anchor.png', 'input.png',
|
|
2203
|
+
... sendtext="Hello World", delay=1.0)
|
|
2204
|
+
"""
|
|
2205
|
+
if backtrack:
|
|
2206
|
+
return self._execute_with_individual_backtrack(
|
|
2207
|
+
"click_relative_image",
|
|
2208
|
+
self._click_relative_image_internal,
|
|
2209
|
+
anchor_image,
|
|
2210
|
+
target_image,
|
|
2211
|
+
max_distance,
|
|
2212
|
+
confidence,
|
|
2213
|
+
target_region,
|
|
2214
|
+
delay,
|
|
2215
|
+
mouse_button,
|
|
2216
|
+
max_attempts,
|
|
2217
|
+
sendtext,
|
|
2218
|
+
wait_until_found,
|
|
2219
|
+
wait_until_disappears,
|
|
2220
|
+
wait_timeout,
|
|
2221
|
+
backtrack=backtrack,
|
|
2222
|
+
)
|
|
2223
|
+
|
|
2224
|
+
return self._click_relative_image_internal(
|
|
2225
|
+
anchor_image,
|
|
2226
|
+
target_image,
|
|
2227
|
+
max_distance,
|
|
2228
|
+
confidence,
|
|
2229
|
+
target_region,
|
|
2230
|
+
delay,
|
|
2231
|
+
mouse_button,
|
|
2232
|
+
max_attempts,
|
|
2233
|
+
sendtext,
|
|
2234
|
+
wait_until_found,
|
|
2235
|
+
wait_until_disappears,
|
|
2236
|
+
wait_timeout,
|
|
2237
|
+
)
|
|
2238
|
+
|
|
2239
|
+
def _click_relative_image_internal(
|
|
2240
|
+
self,
|
|
2241
|
+
anchor_image,
|
|
2242
|
+
target_image,
|
|
2243
|
+
max_distance,
|
|
2244
|
+
confidence,
|
|
2245
|
+
target_region,
|
|
2246
|
+
delay,
|
|
2247
|
+
mouse_button,
|
|
2248
|
+
max_attempts,
|
|
2249
|
+
sendtext=None,
|
|
2250
|
+
wait_until_found=False,
|
|
2251
|
+
wait_until_disappears=False,
|
|
2252
|
+
wait_timeout=None,
|
|
2253
|
+
):
|
|
2254
|
+
"""Internal implementation of the relative image click."""
|
|
2255
|
+
for attempt in range(max_attempts):
|
|
2256
|
+
# For click_relative_image, wait_until_disappears must wait AFTER the click, not before
|
|
2257
|
+
# So we pass only wait_until_found to find_relative_image
|
|
2258
|
+
location = self.find_relative_image(
|
|
2259
|
+
anchor_image,
|
|
2260
|
+
target_image,
|
|
2261
|
+
max_distance,
|
|
2262
|
+
confidence,
|
|
2263
|
+
target_region,
|
|
2264
|
+
wait_until_found,
|
|
2265
|
+
False,
|
|
2266
|
+
wait_timeout,
|
|
2267
|
+
) # wait_until_disappears=False
|
|
2268
|
+
if location:
|
|
2269
|
+
temp_task = {
|
|
2270
|
+
"mouse_button": mouse_button,
|
|
2271
|
+
"delay": delay,
|
|
2272
|
+
"sendtext": sendtext,
|
|
2273
|
+
"wait_until_disappears": wait_until_disappears,
|
|
2274
|
+
"wait_timeout": wait_timeout,
|
|
2275
|
+
"image": target_image, # So wait_until_disappears knows which image to wait for
|
|
2276
|
+
"_wait_after_click": wait_until_disappears,
|
|
2277
|
+
"_wait_timeout": wait_timeout,
|
|
2278
|
+
"_image_path": target_image,
|
|
2279
|
+
}
|
|
2280
|
+
self.executor._perform_action(temp_task, location)
|
|
2281
|
+
return True
|
|
2282
|
+
|
|
2283
|
+
if attempt < max_attempts - 1:
|
|
2284
|
+
time.sleep(0.5)
|
|
2285
|
+
|
|
2286
|
+
return False
|
|
2287
|
+
|
|
2288
|
+
@_records_in_session("click")
|
|
2289
|
+
def click_coordinates(self, x, y, delay=0, mouse_button="left", backtrack=False):
|
|
2290
|
+
"""
|
|
2291
|
+
Click on specific screen coordinates.
|
|
2292
|
+
|
|
2293
|
+
Args:
|
|
2294
|
+
x (int): X coordinate
|
|
2295
|
+
y (int): Y coordinate
|
|
2296
|
+
delay (float): Delay after the click
|
|
2297
|
+
mouse_button (str): Mouse button ("left", "right", "double", "move_to")
|
|
2298
|
+
backtrack (bool): Whether to use backtracking
|
|
2299
|
+
|
|
2300
|
+
Returns:
|
|
2301
|
+
bool: True if clicked successfully
|
|
2302
|
+
|
|
2303
|
+
Examples:
|
|
2304
|
+
>>> success = bot.click_coordinates(100, 200, delay=1, backtrack=True)
|
|
2305
|
+
"""
|
|
2306
|
+
if backtrack:
|
|
2307
|
+
return self._execute_with_individual_backtrack(
|
|
2308
|
+
"click_coordinates",
|
|
2309
|
+
self._click_coordinates_internal,
|
|
2310
|
+
x,
|
|
2311
|
+
y,
|
|
2312
|
+
delay,
|
|
2313
|
+
mouse_button,
|
|
2314
|
+
backtrack=backtrack,
|
|
2315
|
+
)
|
|
2316
|
+
|
|
2317
|
+
return self._click_coordinates_internal(x, y, delay, mouse_button)
|
|
2318
|
+
|
|
2319
|
+
def _click_coordinates_internal(self, x, y, delay, mouse_button):
|
|
2320
|
+
"""Internal implementation of the coordinate click."""
|
|
2321
|
+
try:
|
|
2322
|
+
location = (x, y, 1, 1) # Create a small region
|
|
2323
|
+
temp_task = {"mouse_button": mouse_button, "delay": delay}
|
|
2324
|
+
self.executor._perform_action(temp_task, location)
|
|
2325
|
+
return True
|
|
2326
|
+
except Exception as e:
|
|
2327
|
+
logger.error(f"Error clicking on coordinates ({x}, {y}): {e}")
|
|
2328
|
+
return False
|
|
2329
|
+
|
|
2330
|
+
@_records_in_session("type_text")
|
|
2331
|
+
def type_text(self, text, interval=0.05, delay=0, backtrack=False):
|
|
2332
|
+
"""
|
|
2333
|
+
Type text with an interval between characters.
|
|
2334
|
+
|
|
2335
|
+
Args:
|
|
2336
|
+
text (str): Text to type
|
|
2337
|
+
interval (float): Interval between characters in seconds
|
|
2338
|
+
delay (float): Delay after typing
|
|
2339
|
+
backtrack (bool): Whether to use backtracking
|
|
2340
|
+
|
|
2341
|
+
Returns:
|
|
2342
|
+
bool: True if typed successfully
|
|
2343
|
+
|
|
2344
|
+
Examples:
|
|
2345
|
+
>>> success = bot.type_text('Hello World!', interval=0.05, backtrack=True)
|
|
2346
|
+
>>> success = bot.type_text('{ctrl}a{del}New text', backtrack=True)
|
|
2347
|
+
"""
|
|
2348
|
+
if backtrack:
|
|
2349
|
+
return self._execute_with_individual_backtrack(
|
|
2350
|
+
"type_text",
|
|
2351
|
+
self._type_text_internal,
|
|
2352
|
+
text,
|
|
2353
|
+
interval,
|
|
2354
|
+
delay,
|
|
2355
|
+
backtrack=backtrack,
|
|
2356
|
+
)
|
|
2357
|
+
|
|
2358
|
+
return self._type_text_internal(text, interval, delay)
|
|
2359
|
+
|
|
2360
|
+
def _type_text_internal(self, text, interval, delay):
|
|
2361
|
+
"""Internal implementation of text typing."""
|
|
2362
|
+
try:
|
|
2363
|
+
# Check whether there are special commands
|
|
2364
|
+
if any(
|
|
2365
|
+
cmd in text.lower() for cmd in ["{ctrl}", "{del}", "{tab}", "{enter}"]
|
|
2366
|
+
):
|
|
2367
|
+
self.executor.keyboard_commander.process_sendtext_command(text)
|
|
2368
|
+
else:
|
|
2369
|
+
self.executor.keyboard_commander.type_text(text, interval)
|
|
2370
|
+
|
|
2371
|
+
if delay > 0:
|
|
2372
|
+
time.sleep(delay)
|
|
2373
|
+
return True
|
|
2374
|
+
except Exception as e:
|
|
2375
|
+
logger.error(f"Error typing text '{text}': {e}")
|
|
2376
|
+
return False
|
|
2377
|
+
|
|
2378
|
+
@_records_in_session("keyboard_command")
|
|
2379
|
+
def keyboard_command(self, command, delay=0, backtrack=False):
|
|
2380
|
+
"""
|
|
2381
|
+
Run a keyboard command.
|
|
2382
|
+
|
|
2383
|
+
Args:
|
|
2384
|
+
command (str): Name of the command to run
|
|
2385
|
+
delay (float): Delay after the command
|
|
2386
|
+
backtrack (bool): Whether to use backtracking
|
|
2387
|
+
|
|
2388
|
+
Returns:
|
|
2389
|
+
bool: True if the command ran successfully
|
|
2390
|
+
|
|
2391
|
+
Examples:
|
|
2392
|
+
>>> success = bot.keyboard_command('Ctrl+S', delay=1, backtrack=True)
|
|
2393
|
+
>>> success = bot.keyboard_command('F7', backtrack=True) # Oracle Forms
|
|
2394
|
+
"""
|
|
2395
|
+
if backtrack:
|
|
2396
|
+
return self._execute_with_individual_backtrack(
|
|
2397
|
+
"keyboard_command",
|
|
2398
|
+
self._keyboard_command_internal,
|
|
2399
|
+
command,
|
|
2400
|
+
delay,
|
|
2401
|
+
backtrack=backtrack,
|
|
2402
|
+
)
|
|
2403
|
+
|
|
2404
|
+
return self._keyboard_command_internal(command, delay)
|
|
2405
|
+
|
|
2406
|
+
def _keyboard_command_internal(self, command, delay):
|
|
2407
|
+
"""Internal implementation of the keyboard command."""
|
|
2408
|
+
try:
|
|
2409
|
+
success = self.executor.keyboard_commander.execute_command(command)
|
|
2410
|
+
if delay > 0:
|
|
2411
|
+
time.sleep(delay)
|
|
2412
|
+
return success
|
|
2413
|
+
except Exception as e:
|
|
2414
|
+
logger.error(f"Error running command '{command}': {e}")
|
|
2415
|
+
return False
|
|
2416
|
+
|
|
2417
|
+
def get_available_keyboard_commands(self):
|
|
2418
|
+
"""
|
|
2419
|
+
Return the list of available keyboard commands.
|
|
2420
|
+
|
|
2421
|
+
Returns:
|
|
2422
|
+
list: List of all available commands
|
|
2423
|
+
|
|
2424
|
+
Examples:
|
|
2425
|
+
>>> commands = bot.get_available_keyboard_commands()
|
|
2426
|
+
>>> print(f"Available commands: {len(commands)}")
|
|
2427
|
+
>>> for cmd in commands[:10]: # Show the first 10
|
|
2428
|
+
... print(f" - {cmd}")
|
|
2429
|
+
"""
|
|
2430
|
+
return self.executor.keyboard_commander.get_available_commands()
|
|
2431
|
+
|
|
2432
|
+
def _process_sendtext(self, sendtext):
|
|
2433
|
+
"""
|
|
2434
|
+
Process special commands in sendtext and type the text.
|
|
2435
|
+
|
|
2436
|
+
Args:
|
|
2437
|
+
sendtext (str): Text with special commands such as {ctrl}a, {del}, etc.
|
|
2438
|
+
"""
|
|
2439
|
+
import time
|
|
2440
|
+
|
|
2441
|
+
import pyautogui
|
|
2442
|
+
import pyperclip
|
|
2443
|
+
|
|
2444
|
+
text_to_write = sendtext
|
|
2445
|
+
commands_processed = True
|
|
2446
|
+
|
|
2447
|
+
while commands_processed:
|
|
2448
|
+
commands_processed = False
|
|
2449
|
+
lower_text = text_to_write.lower()
|
|
2450
|
+
|
|
2451
|
+
if lower_text.startswith("{ctrl}a"):
|
|
2452
|
+
pyautogui.hotkey("ctrl", "a")
|
|
2453
|
+
text_to_write = text_to_write[7:] # Remove '{ctrl}a'
|
|
2454
|
+
commands_processed = True
|
|
2455
|
+
time.sleep(0.1)
|
|
2456
|
+
|
|
2457
|
+
elif lower_text.startswith("{del}"):
|
|
2458
|
+
pyautogui.press("delete")
|
|
2459
|
+
text_to_write = text_to_write[5:] # Remove '{del}'
|
|
2460
|
+
commands_processed = True
|
|
2461
|
+
time.sleep(0.1)
|
|
2462
|
+
|
|
2463
|
+
elif lower_text.startswith("{tab}"):
|
|
2464
|
+
pyautogui.press("tab")
|
|
2465
|
+
text_to_write = text_to_write[5:] # Remove '{tab}'
|
|
2466
|
+
commands_processed = True
|
|
2467
|
+
time.sleep(0.1)
|
|
2468
|
+
|
|
2469
|
+
elif lower_text.startswith("{enter}"):
|
|
2470
|
+
pyautogui.press("enter")
|
|
2471
|
+
text_to_write = text_to_write[7:] # Remove '{enter}'
|
|
2472
|
+
commands_processed = True
|
|
2473
|
+
time.sleep(0.1)
|
|
2474
|
+
|
|
2475
|
+
# Type the remaining text using the clipboard for greater reliability
|
|
2476
|
+
if text_to_write:
|
|
2477
|
+
try:
|
|
2478
|
+
pyperclip.copy(text_to_write)
|
|
2479
|
+
pyautogui.hotkey("ctrl", "v")
|
|
2480
|
+
except Exception:
|
|
2481
|
+
# Fallback to direct typing
|
|
2482
|
+
pyautogui.write(text_to_write)
|
|
2483
|
+
time.sleep(0.1)
|
|
2484
|
+
|
|
2485
|
+
def execute_with_backtrack_between_tasks(self, tasks_list):
|
|
2486
|
+
"""
|
|
2487
|
+
Execute multiple individual tasks with real backtracking between them.
|
|
2488
|
+
|
|
2489
|
+
When a task fails, it goes back to the previous one, runs it,
|
|
2490
|
+
and then returns to try the failed one again.
|
|
2491
|
+
|
|
2492
|
+
Args:
|
|
2493
|
+
tasks_list (list): List of dictionaries with individual task settings
|
|
2494
|
+
Each item must have: {'type': 'text'|'image', 'params': {...}}
|
|
2495
|
+
|
|
2496
|
+
Returns:
|
|
2497
|
+
list: List of boolean results
|
|
2498
|
+
"""
|
|
2499
|
+
if not tasks_list:
|
|
2500
|
+
return []
|
|
2501
|
+
|
|
2502
|
+
results = []
|
|
2503
|
+
i = 0
|
|
2504
|
+
backtrack_stack = []
|
|
2505
|
+
task_failures = {}
|
|
2506
|
+
|
|
2507
|
+
logger.info(
|
|
2508
|
+
f"🚀 Starting execution with backtracking across {len(tasks_list)} individual tasks"
|
|
2509
|
+
)
|
|
2510
|
+
|
|
2511
|
+
while i < len(tasks_list):
|
|
2512
|
+
task_config = tasks_list[i]
|
|
2513
|
+
task_type = task_config.get("type")
|
|
2514
|
+
params = task_config.get("params", {})
|
|
2515
|
+
backtrack_enabled = params.get("backtrack", True)
|
|
2516
|
+
|
|
2517
|
+
logger.info(
|
|
2518
|
+
f"📋 Running task {i + 1}/{len(tasks_list)} - Type: {task_type}"
|
|
2519
|
+
)
|
|
2520
|
+
|
|
2521
|
+
success = False
|
|
2522
|
+
|
|
2523
|
+
try:
|
|
2524
|
+
if task_type == "text":
|
|
2525
|
+
# Remove backtrack from params to avoid recursion
|
|
2526
|
+
text_params = params.copy()
|
|
2527
|
+
text_params.pop("backtrack", None)
|
|
2528
|
+
success = self.click_text(**text_params)
|
|
2529
|
+
|
|
2530
|
+
elif task_type == "image":
|
|
2531
|
+
# Remove backtrack from params to avoid recursion
|
|
2532
|
+
image_params = params.copy()
|
|
2533
|
+
image_params.pop("backtrack", None)
|
|
2534
|
+
success = self.click_image(**image_params)
|
|
2535
|
+
|
|
2536
|
+
else:
|
|
2537
|
+
logger.error(f"Unknown task type: {task_type}")
|
|
2538
|
+
success = False
|
|
2539
|
+
|
|
2540
|
+
except Exception as e:
|
|
2541
|
+
logger.error(f"Error running task {i + 1}: {e}")
|
|
2542
|
+
success = False
|
|
2543
|
+
|
|
2544
|
+
# Adjust the results list
|
|
2545
|
+
while len(results) <= i:
|
|
2546
|
+
results.append(False)
|
|
2547
|
+
results[i] = success
|
|
2548
|
+
|
|
2549
|
+
if success:
|
|
2550
|
+
logger.info(f"✓ Task {i + 1}/{len(tasks_list)} executed successfully!")
|
|
2551
|
+
|
|
2552
|
+
# Check whether there are pending backtracks on the stack
|
|
2553
|
+
if backtrack_stack:
|
|
2554
|
+
# Pop the item from the stack and go back to the original task that failed
|
|
2555
|
+
original_failed_task = backtrack_stack.pop()
|
|
2556
|
+
logger.info(
|
|
2557
|
+
f"🔄 Returning to task {original_failed_task + 1} that originally failed after backtrack"
|
|
2558
|
+
)
|
|
2559
|
+
i = original_failed_task
|
|
2560
|
+
# Reset the failure attempts for the original task
|
|
2561
|
+
if i in task_failures:
|
|
2562
|
+
task_failures[i] = 0
|
|
2563
|
+
else:
|
|
2564
|
+
# Normal behavior: advance to the next task
|
|
2565
|
+
i += 1
|
|
2566
|
+
|
|
2567
|
+
else:
|
|
2568
|
+
# Task failed, check backtracking
|
|
2569
|
+
if backtrack_enabled and i > 0:
|
|
2570
|
+
# Manage backtracking
|
|
2571
|
+
task_failures.setdefault(i, 0)
|
|
2572
|
+
task_failures[i] += 1
|
|
2573
|
+
|
|
2574
|
+
if task_failures[i] <= 2: # Limit backtrack attempts
|
|
2575
|
+
# Add the current task to the backtrack stack (to return later)
|
|
2576
|
+
if i not in backtrack_stack: # Avoid duplicates
|
|
2577
|
+
backtrack_stack.append(i)
|
|
2578
|
+
logger.info(
|
|
2579
|
+
f"📌 Task {i + 1} added to the backtrack stack for later re-execution"
|
|
2580
|
+
)
|
|
2581
|
+
|
|
2582
|
+
prev_task_type = tasks_list[i - 1].get("type", "unknown")
|
|
2583
|
+
logger.info(
|
|
2584
|
+
f"✗ Task {i + 1}/{len(tasks_list)} ({task_type}) failed. "
|
|
2585
|
+
f"BACKTRACKING to task {i}/{len(tasks_list)} ({prev_task_type})"
|
|
2586
|
+
)
|
|
2587
|
+
i -= 1 # Go back to the previous task (THIS IS THE KEY LINE!)
|
|
2588
|
+
else:
|
|
2589
|
+
logger.info(
|
|
2590
|
+
f"✗ Task {i + 1}/{len(tasks_list)} failed "
|
|
2591
|
+
f"after multiple backtracking attempts. Advancing."
|
|
2592
|
+
)
|
|
2593
|
+
# Remove from the stack if it is there
|
|
2594
|
+
if i in backtrack_stack:
|
|
2595
|
+
backtrack_stack.remove(i)
|
|
2596
|
+
i += 1
|
|
2597
|
+
else:
|
|
2598
|
+
# No backtrack or first task
|
|
2599
|
+
if backtrack_enabled:
|
|
2600
|
+
logger.info(
|
|
2601
|
+
f"✗ Task {i + 1}/{len(tasks_list)} failed "
|
|
2602
|
+
f"but it is the first task. Advancing."
|
|
2603
|
+
)
|
|
2604
|
+
else:
|
|
2605
|
+
logger.info(
|
|
2606
|
+
f"✗ Task {i + 1}/{len(tasks_list)} failed "
|
|
2607
|
+
f"and has 'backtrack': False. Advancing."
|
|
2608
|
+
)
|
|
2609
|
+
i += 1
|
|
2610
|
+
|
|
2611
|
+
successful_tasks = sum(1 for r in results if r)
|
|
2612
|
+
logger.info(
|
|
2613
|
+
f"🏁 Execution finished: {successful_tasks}/{len(results)} successful tasks"
|
|
2614
|
+
)
|
|
2615
|
+
|
|
2616
|
+
return results
|
|
2617
|
+
|
|
2618
|
+
|
|
2619
|
+
# Backward-compatibility alias (previous branding: bot-vision-suite / import bot_vision)
|
|
2620
|
+
BotVision = VisusBot
|
|
2621
|
+
|
|
2622
|
+
|
|
2623
|
+
# NEW CONVENIENCE FUNCTIONS WITH INDIVIDUAL BACKTRACK
|
|
2624
|
+
|
|
2625
|
+
|
|
2626
|
+
def start_individual_session(config=None):
|
|
2627
|
+
"""
|
|
2628
|
+
Start an individual task session with backtracking.
|
|
2629
|
+
|
|
2630
|
+
Args:
|
|
2631
|
+
config (dict, optional): Custom configuration
|
|
2632
|
+
|
|
2633
|
+
Returns:
|
|
2634
|
+
VisusBot: Instance configured for individual backtracking
|
|
2635
|
+
"""
|
|
2636
|
+
bot = VisusBot(config)
|
|
2637
|
+
bot.start_task_session()
|
|
2638
|
+
return bot
|
|
2639
|
+
|
|
2640
|
+
|
|
2641
|
+
def run_individual_tasks_with_backtrack(task_functions, config=None):
|
|
2642
|
+
"""
|
|
2643
|
+
Run a list of individual task functions with automatic backtracking.
|
|
2644
|
+
|
|
2645
|
+
Args:
|
|
2646
|
+
task_functions (list): List of tuples (function, args, kwargs)
|
|
2647
|
+
config (dict, optional): Custom configuration
|
|
2648
|
+
|
|
2649
|
+
Returns:
|
|
2650
|
+
tuple: (successes, total, bot_instance)
|
|
2651
|
+
|
|
2652
|
+
Examples:
|
|
2653
|
+
>>> tasks = [
|
|
2654
|
+
... (lambda bot: bot.click_image('btn.png', backtrack=True), (), {}),
|
|
2655
|
+
... (lambda bot: bot.click_text('Save', backtrack=True), (), {}),
|
|
2656
|
+
... ]
|
|
2657
|
+
>>> success, total, bot = run_individual_tasks_with_backtrack(tasks)
|
|
2658
|
+
"""
|
|
2659
|
+
bot = VisusBot(config)
|
|
2660
|
+
bot.start_task_session()
|
|
2661
|
+
|
|
2662
|
+
results = []
|
|
2663
|
+
for i, (func, args, kwargs) in enumerate(task_functions):
|
|
2664
|
+
try:
|
|
2665
|
+
result = func(bot, *args, **kwargs)
|
|
2666
|
+
results.append(result)
|
|
2667
|
+
logger.info(f"Task {i + 1}: {'✓ Success' if result else '✗ Failed'}")
|
|
2668
|
+
except Exception as e:
|
|
2669
|
+
logger.error(f"Error in task {i + 1}: {e}")
|
|
2670
|
+
results.append(False)
|
|
2671
|
+
|
|
2672
|
+
successful, total = bot.end_task_session()
|
|
2673
|
+
return successful, total, bot
|
|
2674
|
+
|
|
2675
|
+
|
|
2676
|
+
# Main function compatible with legacy code
|
|
2677
|
+
def execute_tasks(tasks, config=None):
|
|
2678
|
+
"""
|
|
2679
|
+
Main function to run a list of tasks.
|
|
2680
|
+
|
|
2681
|
+
Supports both simple lists and lists of lists (multiple sets).
|
|
2682
|
+
This is the function that users migrated from the original code will use.
|
|
2683
|
+
|
|
2684
|
+
Args:
|
|
2685
|
+
tasks (list): List of automation tasks or list of lists
|
|
2686
|
+
config (dict, optional): Custom configuration
|
|
2687
|
+
|
|
2688
|
+
Returns:
|
|
2689
|
+
list: List of TaskResult or list of lists of TaskResult
|
|
2690
|
+
|
|
2691
|
+
Examples:
|
|
2692
|
+
Simple list:
|
|
2693
|
+
>>> tasks = [{'text': 'Login', 'region': (100, 100, 500, 300)}]
|
|
2694
|
+
>>> results = execute_tasks(tasks)
|
|
2695
|
+
|
|
2696
|
+
Multiple lists:
|
|
2697
|
+
>>> task_sets = [
|
|
2698
|
+
... [{'text': 'Login', 'region': (100, 100, 500, 300)}],
|
|
2699
|
+
... [{'text': 'Save', 'region': (200, 200, 600, 400)}]
|
|
2700
|
+
... ]
|
|
2701
|
+
>>> results = execute_tasks(task_sets)
|
|
2702
|
+
"""
|
|
2703
|
+
if not isinstance(tasks, list):
|
|
2704
|
+
logger.error(f"Tasks must be a list. Received type: {type(tasks)}")
|
|
2705
|
+
return []
|
|
2706
|
+
|
|
2707
|
+
if not tasks:
|
|
2708
|
+
logger.info("Empty task list.")
|
|
2709
|
+
return []
|
|
2710
|
+
|
|
2711
|
+
# Check whether it is a list of lists
|
|
2712
|
+
if isinstance(tasks[0], list):
|
|
2713
|
+
# Multiple task sets
|
|
2714
|
+
logger.info(
|
|
2715
|
+
f"Detected multiple task sets ({len(tasks)} sets). Running sequentially."
|
|
2716
|
+
)
|
|
2717
|
+
|
|
2718
|
+
all_results = []
|
|
2719
|
+
for i, task_list in enumerate(tasks):
|
|
2720
|
+
if isinstance(task_list, list):
|
|
2721
|
+
logger.info(
|
|
2722
|
+
f"--- Starting task set {i + 1}/{len(tasks)} ({len(task_list)} tasks) ---"
|
|
2723
|
+
)
|
|
2724
|
+
bot = VisusBot(config)
|
|
2725
|
+
results = bot.execute_tasks(task_list)
|
|
2726
|
+
all_results.append(results)
|
|
2727
|
+
logger.info(f"--- Finished task set {i + 1}/{len(tasks)} ---")
|
|
2728
|
+
else:
|
|
2729
|
+
logger.warning(
|
|
2730
|
+
f"Item {i} in the main list is not a list of tasks. Skipping."
|
|
2731
|
+
)
|
|
2732
|
+
all_results.append([])
|
|
2733
|
+
|
|
2734
|
+
return all_results
|
|
2735
|
+
else:
|
|
2736
|
+
# Simple list of tasks
|
|
2737
|
+
bot = VisusBot(config)
|
|
2738
|
+
return bot.execute_tasks(tasks)
|
|
2739
|
+
|
|
2740
|
+
|
|
2741
|
+
# Additional function for full compatibility with the original click_images
|
|
2742
|
+
def click_images(tasks, default_confidence=0.9, default_margin=50):
|
|
2743
|
+
"""
|
|
2744
|
+
Function for full compatibility with legacy code.
|
|
2745
|
+
|
|
2746
|
+
This function replicates exactly the behavior of the click_images function
|
|
2747
|
+
from the original bot_vision.py, including support for lists of lists.
|
|
2748
|
+
|
|
2749
|
+
Args:
|
|
2750
|
+
tasks: List of tasks or list of lists of tasks
|
|
2751
|
+
default_confidence (float): Default confidence for image detection
|
|
2752
|
+
default_margin (int): Default margin (kept for compatibility)
|
|
2753
|
+
|
|
2754
|
+
Returns:
|
|
2755
|
+
list: List of results
|
|
2756
|
+
"""
|
|
2757
|
+
# Build a configuration with compatible default values
|
|
2758
|
+
config = {
|
|
2759
|
+
"default_confidence": default_confidence,
|
|
2760
|
+
"default_margin": default_margin,
|
|
2761
|
+
"tesseract_path": r"C:\Program Files\Tesseract-OCR\tesseract.exe",
|
|
2762
|
+
"tessdata_path": r"C:\Program Files\Tesseract-OCR\tessdata",
|
|
2763
|
+
}
|
|
2764
|
+
|
|
2765
|
+
return execute_tasks(tasks, config)
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
# Standalone functions for 100% compatibility with the original code
|
|
2769
|
+
def locate_image_with_retry(
|
|
2770
|
+
image_path: str, region=None, confidence=0.9, max_attempts=3, scales=None
|
|
2771
|
+
):
|
|
2772
|
+
"""
|
|
2773
|
+
Standalone function that replicates exactly the original function from bot_vision.py.
|
|
2774
|
+
Tries to locate an image with different scales and confidence levels.
|
|
2775
|
+
"""
|
|
2776
|
+
executor = TaskExecutor()
|
|
2777
|
+
return executor._locate_image_with_retry(
|
|
2778
|
+
image_path, region, confidence, max_attempts, scales
|
|
2779
|
+
)
|
|
2780
|
+
|
|
2781
|
+
|
|
2782
|
+
# Standalone convenience functions - NEW FEATURES
|
|
2783
|
+
def find_relative_image(
|
|
2784
|
+
anchor_image,
|
|
2785
|
+
target_image,
|
|
2786
|
+
max_distance=200,
|
|
2787
|
+
confidence=0.9,
|
|
2788
|
+
target_region=None,
|
|
2789
|
+
wait_until_found=False,
|
|
2790
|
+
wait_until_disappears=False,
|
|
2791
|
+
wait_timeout=None,
|
|
2792
|
+
):
|
|
2793
|
+
"""
|
|
2794
|
+
Standalone function to find a relative image.
|
|
2795
|
+
|
|
2796
|
+
Args:
|
|
2797
|
+
anchor_image (str): Path to the anchor image
|
|
2798
|
+
target_image (str): Path to the target image
|
|
2799
|
+
max_distance (int): Maximum distance in pixels
|
|
2800
|
+
confidence (float): Confidence level
|
|
2801
|
+
target_region (tuple, optional): Region to search for the target
|
|
2802
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
2803
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
2804
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
2805
|
+
|
|
2806
|
+
Returns:
|
|
2807
|
+
tuple: Location of the image or None
|
|
2808
|
+
|
|
2809
|
+
Examples:
|
|
2810
|
+
>>> location = find_relative_image('anchor.png', 'target.png')
|
|
2811
|
+
"""
|
|
2812
|
+
bot = VisusBot()
|
|
2813
|
+
return bot.find_relative_image(
|
|
2814
|
+
anchor_image,
|
|
2815
|
+
target_image,
|
|
2816
|
+
max_distance,
|
|
2817
|
+
confidence,
|
|
2818
|
+
target_region,
|
|
2819
|
+
wait_until_found,
|
|
2820
|
+
wait_until_disappears,
|
|
2821
|
+
wait_timeout,
|
|
2822
|
+
)
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
def click_relative_image(
|
|
2826
|
+
anchor_image,
|
|
2827
|
+
target_image,
|
|
2828
|
+
max_distance=200,
|
|
2829
|
+
confidence=0.9,
|
|
2830
|
+
target_region=None,
|
|
2831
|
+
delay=0,
|
|
2832
|
+
mouse_button="left",
|
|
2833
|
+
backtrack=False,
|
|
2834
|
+
wait_until_found=False,
|
|
2835
|
+
wait_until_disappears=False,
|
|
2836
|
+
wait_timeout=None,
|
|
2837
|
+
):
|
|
2838
|
+
"""
|
|
2839
|
+
Standalone function to click on a relative image.
|
|
2840
|
+
|
|
2841
|
+
Args:
|
|
2842
|
+
anchor_image (str): Path to the anchor image
|
|
2843
|
+
target_image (str): Path to the target image
|
|
2844
|
+
max_distance (int): Maximum distance in pixels
|
|
2845
|
+
confidence (float): Confidence level (0.0-1.0)
|
|
2846
|
+
target_region (tuple, optional): Region to search for the target (x, y, width, height)
|
|
2847
|
+
delay (float): Delay after the click in seconds
|
|
2848
|
+
mouse_button (str): Mouse click type:
|
|
2849
|
+
• "left" - Simple left click (default)
|
|
2850
|
+
• "right" - Right click
|
|
2851
|
+
• "double" or "double left" - Double left click
|
|
2852
|
+
• "move_to" - Just moves the mouse without clicking
|
|
2853
|
+
backtrack (bool): Whether to use backtracking on failure
|
|
2854
|
+
|
|
2855
|
+
Returns:
|
|
2856
|
+
bool: True if clicked successfully, False otherwise
|
|
2857
|
+
|
|
2858
|
+
Examples:
|
|
2859
|
+
>>> # Simple left click
|
|
2860
|
+
>>> success = click_relative_image('anchor.png', 'target.png')
|
|
2861
|
+
|
|
2862
|
+
>>> # Right click
|
|
2863
|
+
>>> success = click_relative_image('anchor.png', 'target.png',
|
|
2864
|
+
... mouse_button="right", backtrack=True)
|
|
2865
|
+
|
|
2866
|
+
>>> # Double left click
|
|
2867
|
+
>>> success = click_relative_image('anchor.png', 'target.png',
|
|
2868
|
+
... mouse_button="double", delay=1.0)
|
|
2869
|
+
|
|
2870
|
+
>>> # Just move the mouse to the position (without clicking)
|
|
2871
|
+
>>> success = click_relative_image('anchor.png', 'target.png',
|
|
2872
|
+
... mouse_button="move_to", delay=0.5)
|
|
2873
|
+
|
|
2874
|
+
Additional args v1.3.0+:
|
|
2875
|
+
wait_until_found (bool): Waits for the image to appear automatically
|
|
2876
|
+
wait_until_disappears (bool): Waits for the image to disappear
|
|
2877
|
+
wait_timeout (int, optional): Specific timeout (overrides the global one)
|
|
2878
|
+
"""
|
|
2879
|
+
bot = VisusBot()
|
|
2880
|
+
return bot.click_relative_image(
|
|
2881
|
+
anchor_image,
|
|
2882
|
+
target_image,
|
|
2883
|
+
max_distance,
|
|
2884
|
+
confidence,
|
|
2885
|
+
target_region,
|
|
2886
|
+
delay,
|
|
2887
|
+
mouse_button,
|
|
2888
|
+
backtrack,
|
|
2889
|
+
3,
|
|
2890
|
+
wait_until_found,
|
|
2891
|
+
wait_until_disappears,
|
|
2892
|
+
wait_timeout,
|
|
2893
|
+
)
|
|
2894
|
+
|
|
2895
|
+
|
|
2896
|
+
def click_coordinates(x, y, delay=0, mouse_button="left", backtrack=False):
|
|
2897
|
+
"""
|
|
2898
|
+
Standalone function to click on specific coordinates.
|
|
2899
|
+
|
|
2900
|
+
Args:
|
|
2901
|
+
x (int): X coordinate
|
|
2902
|
+
y (int): Y coordinate
|
|
2903
|
+
delay (float): Delay after the click in seconds
|
|
2904
|
+
mouse_button (str): Mouse click type:
|
|
2905
|
+
• "left" - Simple left click (default)
|
|
2906
|
+
• "right" - Right click
|
|
2907
|
+
• "double" or "double left" - Double left click
|
|
2908
|
+
• "move_to" - Just moves the mouse without clicking
|
|
2909
|
+
backtrack (bool): Whether to use backtracking on failure
|
|
2910
|
+
|
|
2911
|
+
Returns:
|
|
2912
|
+
bool: True if clicked successfully, False otherwise
|
|
2913
|
+
|
|
2914
|
+
Examples:
|
|
2915
|
+
>>> # Simple click
|
|
2916
|
+
>>> success = click_coordinates(100, 200, delay=1)
|
|
2917
|
+
|
|
2918
|
+
>>> # Right click
|
|
2919
|
+
>>> success = click_coordinates(100, 200, mouse_button="right", backtrack=True)
|
|
2920
|
+
|
|
2921
|
+
>>> # Just move the mouse to the position (without clicking)
|
|
2922
|
+
>>> success = click_coordinates(100, 200, mouse_button="move_to", delay=0.5)
|
|
2923
|
+
"""
|
|
2924
|
+
bot = VisusBot()
|
|
2925
|
+
return bot.click_coordinates(x, y, delay, mouse_button, backtrack)
|
|
2926
|
+
|
|
2927
|
+
|
|
2928
|
+
def type_text_standalone(text, interval=0.05, delay=0, backtrack=False):
|
|
2929
|
+
"""
|
|
2930
|
+
Standalone function to type text.
|
|
2931
|
+
|
|
2932
|
+
Args:
|
|
2933
|
+
text (str): Text to type
|
|
2934
|
+
interval (float): Interval between characters
|
|
2935
|
+
delay (float): Delay after typing
|
|
2936
|
+
backtrack (bool): Whether to use backtracking
|
|
2937
|
+
|
|
2938
|
+
Returns:
|
|
2939
|
+
bool: True if typed successfully
|
|
2940
|
+
|
|
2941
|
+
Examples:
|
|
2942
|
+
>>> success = type_text_standalone('Hello World!', backtrack=True)
|
|
2943
|
+
>>> success = type_text_standalone('{ctrl}a{del}New text', backtrack=True)
|
|
2944
|
+
"""
|
|
2945
|
+
bot = VisusBot()
|
|
2946
|
+
return bot.type_text(text, interval, delay, backtrack)
|
|
2947
|
+
|
|
2948
|
+
|
|
2949
|
+
def keyboard_command_standalone(command, delay=0, backtrack=False):
|
|
2950
|
+
"""
|
|
2951
|
+
Standalone function to run a keyboard command.
|
|
2952
|
+
|
|
2953
|
+
Args:
|
|
2954
|
+
command (str): Command to run
|
|
2955
|
+
delay (float): Delay after the command
|
|
2956
|
+
backtrack (bool): Whether to use backtracking
|
|
2957
|
+
|
|
2958
|
+
Returns:
|
|
2959
|
+
bool: True if it ran successfully
|
|
2960
|
+
|
|
2961
|
+
Examples:
|
|
2962
|
+
>>> success = keyboard_command_standalone('Ctrl+S', delay=1, backtrack=True)
|
|
2963
|
+
>>> success = keyboard_command_standalone('F7', backtrack=True)
|
|
2964
|
+
"""
|
|
2965
|
+
bot = VisusBot()
|
|
2966
|
+
return bot.keyboard_command(command, delay, backtrack)
|
|
2967
|
+
|
|
2968
|
+
|
|
2969
|
+
def get_available_keyboard_commands():
|
|
2970
|
+
"""
|
|
2971
|
+
Standalone function to get the available keyboard commands.
|
|
2972
|
+
|
|
2973
|
+
Returns:
|
|
2974
|
+
list: List of available commands
|
|
2975
|
+
|
|
2976
|
+
Examples:
|
|
2977
|
+
>>> commands = get_available_keyboard_commands()
|
|
2978
|
+
>>> print(f"Total commands: {len(commands)}")
|
|
2979
|
+
"""
|
|
2980
|
+
bot = VisusBot()
|
|
2981
|
+
return bot.get_available_keyboard_commands()
|
|
2982
|
+
|
|
2983
|
+
|
|
2984
|
+
def extract_text_from_region_standalone(
|
|
2985
|
+
region=None,
|
|
2986
|
+
filter_type="both",
|
|
2987
|
+
confidence_threshold=50.0,
|
|
2988
|
+
return_full_data=False,
|
|
2989
|
+
max_attempts=3,
|
|
2990
|
+
backtrack=False,
|
|
2991
|
+
config=None,
|
|
2992
|
+
):
|
|
2993
|
+
"""
|
|
2994
|
+
Standalone function to extract text from a specific region of the screen.
|
|
2995
|
+
|
|
2996
|
+
Args:
|
|
2997
|
+
region (tuple, optional): (x, y, width, height) of the region to extract text from.
|
|
2998
|
+
If None, extracts from the whole screen
|
|
2999
|
+
filter_type (str): Filter type ("numbers", "letters", "both")
|
|
3000
|
+
confidence_threshold (float): Minimum confidence threshold (0-100)
|
|
3001
|
+
return_full_data (bool): If True, returns full data with coordinates
|
|
3002
|
+
max_attempts (int): Maximum number of attempts if backtrack=True
|
|
3003
|
+
backtrack (bool): Whether to try multiple times with adjustments
|
|
3004
|
+
config (dict or BotVisionConfig, optional): Custom configuration
|
|
3005
|
+
|
|
3006
|
+
Returns:
|
|
3007
|
+
If return_full_data=False:
|
|
3008
|
+
list: List of strings with all the text found
|
|
3009
|
+
If return_full_data=True:
|
|
3010
|
+
list: List of dictionaries with full data
|
|
3011
|
+
|
|
3012
|
+
Examples:
|
|
3013
|
+
Simple extraction:
|
|
3014
|
+
>>> texts = extract_text_from_region_standalone(region=(100, 100, 400, 300))
|
|
3015
|
+
>>> print("Texts found:", texts)
|
|
3016
|
+
|
|
3017
|
+
Extraction with full data:
|
|
3018
|
+
>>> data = extract_text_from_region_standalone(
|
|
3019
|
+
... region=(100, 100, 400, 300),
|
|
3020
|
+
... return_full_data=True,
|
|
3021
|
+
... confidence_threshold=70.0,
|
|
3022
|
+
... backtrack=True
|
|
3023
|
+
... )
|
|
3024
|
+
>>> for item in data:
|
|
3025
|
+
... print(f"'{item['text']}' - Confidence: {item['confidence']:.1f}%")
|
|
3026
|
+
|
|
3027
|
+
Numbers only:
|
|
3028
|
+
>>> numbers = extract_text_from_region_standalone(
|
|
3029
|
+
... region=(50, 50, 200, 100),
|
|
3030
|
+
... filter_type="numbers"
|
|
3031
|
+
... )
|
|
3032
|
+
"""
|
|
3033
|
+
bot = VisusBot(config=config)
|
|
3034
|
+
return bot.extract_text_from_region(
|
|
3035
|
+
region=region,
|
|
3036
|
+
filter_type=filter_type,
|
|
3037
|
+
confidence_threshold=confidence_threshold,
|
|
3038
|
+
return_full_data=return_full_data,
|
|
3039
|
+
max_attempts=max_attempts,
|
|
3040
|
+
backtrack=backtrack,
|
|
3041
|
+
)
|
|
3042
|
+
|
|
3043
|
+
|
|
3044
|
+
# Function for full compatibility - task execution with multiple-list support
|
|
3045
|
+
def run_automation(tasks, default_confidence=0.9, default_margin=50):
|
|
3046
|
+
"""
|
|
3047
|
+
Function that replicates the exact behavior of the if __name__ == '__main__' block of the original bot_vision.py.
|
|
3048
|
+
Supports both a simple list and a list of lists (multiple task sets).
|
|
3049
|
+
|
|
3050
|
+
Args:
|
|
3051
|
+
tasks: List of tasks or list of lists of tasks (as in the original)
|
|
3052
|
+
default_confidence (float): Default confidence
|
|
3053
|
+
default_margin (int): Default margin
|
|
3054
|
+
"""
|
|
3055
|
+
import logging
|
|
3056
|
+
|
|
3057
|
+
# Check whether tasks is a list and not empty
|
|
3058
|
+
if isinstance(tasks, list) and tasks:
|
|
3059
|
+
# Check whether the first element is also a list (indicating a list of lists)
|
|
3060
|
+
if isinstance(tasks[0], list):
|
|
3061
|
+
logging.info(
|
|
3062
|
+
f"Detected multiple task lists ({len(tasks)} lists). Executing sequentially."
|
|
3063
|
+
)
|
|
3064
|
+
# Iterate through each task list
|
|
3065
|
+
for i, task_list in enumerate(tasks):
|
|
3066
|
+
if isinstance(task_list, list):
|
|
3067
|
+
logging.info(
|
|
3068
|
+
f"--- Starting task list {i + 1}/{len(tasks)} ({len(task_list)} tasks) ---"
|
|
3069
|
+
)
|
|
3070
|
+
click_images(
|
|
3071
|
+
task_list, default_confidence, default_margin
|
|
3072
|
+
) # Pass the individual list to the function
|
|
3073
|
+
logging.info(f"--- Finished task list {i + 1}/{len(tasks)} ---")
|
|
3074
|
+
else:
|
|
3075
|
+
logging.warning(
|
|
3076
|
+
f"Item {i} in the main list is not a list of tasks. Skipping."
|
|
3077
|
+
)
|
|
3078
|
+
else:
|
|
3079
|
+
# Assume it is a single flat list of tasks
|
|
3080
|
+
logging.info("Detected a single task list. Executing.")
|
|
3081
|
+
click_images(tasks, default_confidence, default_margin)
|
|
3082
|
+
elif isinstance(tasks, list) and not tasks:
|
|
3083
|
+
logging.info("The imported 'tasks' list is empty. Nothing to execute.")
|
|
3084
|
+
else:
|
|
3085
|
+
logging.error(
|
|
3086
|
+
f"The imported 'tasks' is not a list. Type: {type(tasks)}. Cannot execute."
|
|
3087
|
+
)
|
|
3088
|
+
|
|
3089
|
+
|
|
3090
|
+
# Initial library setup
|
|
3091
|
+
def _setup_library():
|
|
3092
|
+
"""Initial library setup."""
|
|
3093
|
+
try:
|
|
3094
|
+
# Configure logging if it is not already configured
|
|
3095
|
+
if not logging.getLogger().handlers:
|
|
3096
|
+
logging.basicConfig(
|
|
3097
|
+
level=logging.INFO, format="[%(levelname)s] %(name)s: %(message)s"
|
|
3098
|
+
)
|
|
3099
|
+
|
|
3100
|
+
# Try to configure the library with the default configuration
|
|
3101
|
+
get_default_config()
|
|
3102
|
+
logger.debug(f"visus.desktop v{__version__} initialized")
|
|
3103
|
+
|
|
3104
|
+
except Exception as e:
|
|
3105
|
+
logger.warning(f"Warning during library initialization: {e}")
|
|
3106
|
+
|
|
3107
|
+
|
|
3108
|
+
# Run the initial setup
|
|
3109
|
+
_setup_library()
|
|
3110
|
+
|
|
3111
|
+
|
|
3112
|
+
# Standalone convenience functions (create a temporary VisusBot instance)
|
|
3113
|
+
def find_text(
|
|
3114
|
+
text,
|
|
3115
|
+
region=None,
|
|
3116
|
+
filter_type="both",
|
|
3117
|
+
confidence_threshold=75.0,
|
|
3118
|
+
occurrence=1,
|
|
3119
|
+
max_attempts=3,
|
|
3120
|
+
config=None,
|
|
3121
|
+
wait_until_found=False,
|
|
3122
|
+
wait_until_disappears=False,
|
|
3123
|
+
wait_timeout=None,
|
|
3124
|
+
):
|
|
3125
|
+
"""
|
|
3126
|
+
Search for text on screen using a temporary VisusBot instance.
|
|
3127
|
+
|
|
3128
|
+
Args:
|
|
3129
|
+
text (str): Text to find
|
|
3130
|
+
region (tuple, optional): Search region (x, y, width, height)
|
|
3131
|
+
filter_type (str): Filter type ("letters", "numbers", "both")
|
|
3132
|
+
confidence_threshold (float): OCR confidence threshold
|
|
3133
|
+
occurrence (int): Which occurrence to look for (1=first)
|
|
3134
|
+
max_attempts (int): Maximum number of attempts
|
|
3135
|
+
config (dict, optional): Custom configuration
|
|
3136
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
|
|
3137
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
|
|
3138
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
3139
|
+
|
|
3140
|
+
Returns:
|
|
3141
|
+
tuple: Found coordinates or None
|
|
3142
|
+
"""
|
|
3143
|
+
bot = VisusBot(config)
|
|
3144
|
+
return bot.find_text(
|
|
3145
|
+
text,
|
|
3146
|
+
region,
|
|
3147
|
+
filter_type,
|
|
3148
|
+
confidence_threshold,
|
|
3149
|
+
occurrence,
|
|
3150
|
+
max_attempts,
|
|
3151
|
+
wait_until_found=wait_until_found,
|
|
3152
|
+
wait_until_disappears=wait_until_disappears,
|
|
3153
|
+
wait_timeout=wait_timeout,
|
|
3154
|
+
)
|
|
3155
|
+
|
|
3156
|
+
|
|
3157
|
+
def click_text(
|
|
3158
|
+
text,
|
|
3159
|
+
region=None,
|
|
3160
|
+
filter_type="both",
|
|
3161
|
+
delay=0,
|
|
3162
|
+
mouse_button="left",
|
|
3163
|
+
occurrence=1,
|
|
3164
|
+
max_attempts=3,
|
|
3165
|
+
sendtext=None,
|
|
3166
|
+
confidence_threshold=None,
|
|
3167
|
+
show_overlay=True,
|
|
3168
|
+
config=None,
|
|
3169
|
+
wait_until_found=False,
|
|
3170
|
+
wait_until_disappears=False,
|
|
3171
|
+
wait_timeout=None,
|
|
3172
|
+
):
|
|
3173
|
+
"""
|
|
3174
|
+
Click on text using a temporary VisusBot instance.
|
|
3175
|
+
|
|
3176
|
+
Args:
|
|
3177
|
+
text (str): Text to click on
|
|
3178
|
+
region (tuple, optional): Search region (x, y, width, height)
|
|
3179
|
+
filter_type (str): Filter type ("letters", "numbers", "both")
|
|
3180
|
+
delay (float): Delay after the click in seconds
|
|
3181
|
+
mouse_button (str): Mouse click type:
|
|
3182
|
+
• "left" - Simple left click (default)
|
|
3183
|
+
• "right" - Right click
|
|
3184
|
+
• "double" or "double left" - Double left click
|
|
3185
|
+
• "move_to" - Just moves the mouse without clicking
|
|
3186
|
+
occurrence (int): Which occurrence to click (1=first)
|
|
3187
|
+
max_attempts (int): Maximum number of attempts
|
|
3188
|
+
sendtext (str, optional): Text to type after the click
|
|
3189
|
+
confidence_threshold (float, optional): Custom confidence threshold
|
|
3190
|
+
show_overlay (bool): Whether to show the red overlay before the click
|
|
3191
|
+
config (dict, optional): Custom configuration
|
|
3192
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
|
|
3193
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
|
|
3194
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
3195
|
+
|
|
3196
|
+
Returns:
|
|
3197
|
+
bool: True if found and clicked, False otherwise
|
|
3198
|
+
"""
|
|
3199
|
+
bot = VisusBot(config)
|
|
3200
|
+
return bot.click_text(
|
|
3201
|
+
text,
|
|
3202
|
+
region,
|
|
3203
|
+
filter_type,
|
|
3204
|
+
delay,
|
|
3205
|
+
mouse_button,
|
|
3206
|
+
occurrence,
|
|
3207
|
+
False,
|
|
3208
|
+
max_attempts,
|
|
3209
|
+
sendtext,
|
|
3210
|
+
confidence_threshold,
|
|
3211
|
+
show_overlay,
|
|
3212
|
+
wait_until_found,
|
|
3213
|
+
wait_until_disappears,
|
|
3214
|
+
wait_timeout,
|
|
3215
|
+
)
|
|
3216
|
+
|
|
3217
|
+
|
|
3218
|
+
def find_image(
|
|
3219
|
+
image_path,
|
|
3220
|
+
region=None,
|
|
3221
|
+
confidence=0.9,
|
|
3222
|
+
max_attempts=3,
|
|
3223
|
+
specific=True,
|
|
3224
|
+
scales=None,
|
|
3225
|
+
config=None,
|
|
3226
|
+
wait_until_found=False,
|
|
3227
|
+
wait_until_disappears=False,
|
|
3228
|
+
wait_timeout=None,
|
|
3229
|
+
occurrence=1,
|
|
3230
|
+
):
|
|
3231
|
+
"""
|
|
3232
|
+
Search for an image on screen using a temporary VisusBot instance.
|
|
3233
|
+
|
|
3234
|
+
Args:
|
|
3235
|
+
image_path (str): Path to the image
|
|
3236
|
+
region (tuple, optional): Search region
|
|
3237
|
+
confidence (float): Confidence level
|
|
3238
|
+
max_attempts (int): Maximum number of attempts
|
|
3239
|
+
specific (bool): If True, searches in the region; if False, searches the whole screen + variations
|
|
3240
|
+
scales (list, optional): List of scales to try
|
|
3241
|
+
config (dict, optional): Custom configuration
|
|
3242
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
3243
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
3244
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
3245
|
+
occurrence (int): Which match to return when the template appears more
|
|
3246
|
+
than once (1=first, reading order top→bottom/left→right)
|
|
3247
|
+
|
|
3248
|
+
Returns:
|
|
3249
|
+
tuple: Coordinates of the image or None
|
|
3250
|
+
"""
|
|
3251
|
+
bot = VisusBot(config)
|
|
3252
|
+
return bot.find_image(
|
|
3253
|
+
image_path,
|
|
3254
|
+
region,
|
|
3255
|
+
confidence,
|
|
3256
|
+
max_attempts,
|
|
3257
|
+
False,
|
|
3258
|
+
specific,
|
|
3259
|
+
wait_until_found,
|
|
3260
|
+
wait_until_disappears,
|
|
3261
|
+
wait_timeout,
|
|
3262
|
+
scales,
|
|
3263
|
+
occurrence=occurrence,
|
|
3264
|
+
)
|
|
3265
|
+
|
|
3266
|
+
|
|
3267
|
+
def click_image(
|
|
3268
|
+
image_path,
|
|
3269
|
+
region=None,
|
|
3270
|
+
confidence=0.9,
|
|
3271
|
+
delay=0,
|
|
3272
|
+
mouse_button="left",
|
|
3273
|
+
max_attempts=3,
|
|
3274
|
+
specific=True,
|
|
3275
|
+
sendtext=None,
|
|
3276
|
+
show_overlay=True,
|
|
3277
|
+
config=None,
|
|
3278
|
+
wait_until_found=False,
|
|
3279
|
+
wait_until_disappears=False,
|
|
3280
|
+
wait_timeout=None,
|
|
3281
|
+
occurrence=1,
|
|
3282
|
+
):
|
|
3283
|
+
"""
|
|
3284
|
+
Click on an image using a temporary VisusBot instance.
|
|
3285
|
+
|
|
3286
|
+
Args:
|
|
3287
|
+
image_path (str): Path to the image
|
|
3288
|
+
region (tuple, optional): Search region (x, y, width, height)
|
|
3289
|
+
confidence (float): Confidence level (0.0-1.0)
|
|
3290
|
+
delay (float): Delay after the click in seconds
|
|
3291
|
+
mouse_button (str): Mouse click type:
|
|
3292
|
+
• "left" - Simple left click (default)
|
|
3293
|
+
• "right" - Right click
|
|
3294
|
+
• "double" or "double left" - Double left click
|
|
3295
|
+
• "move_to" - Just moves the mouse without clicking
|
|
3296
|
+
max_attempts (int): Maximum number of attempts
|
|
3297
|
+
specific (bool): If True, searches in the region; if False, searches the whole screen + variations
|
|
3298
|
+
sendtext (str, optional): Text to type after the click
|
|
3299
|
+
show_overlay (bool): Whether to show the red overlay before the click
|
|
3300
|
+
config (dict, optional): Custom configuration
|
|
3301
|
+
wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
|
|
3302
|
+
wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
|
|
3303
|
+
wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
|
|
3304
|
+
occurrence (int): Which match to click when the template appears more
|
|
3305
|
+
than once (1=first, reading order top→bottom/left→right)
|
|
3306
|
+
|
|
3307
|
+
Returns:
|
|
3308
|
+
bool: True if found and clicked, False otherwise
|
|
3309
|
+
"""
|
|
3310
|
+
bot = VisusBot(config)
|
|
3311
|
+
return bot.click_image(
|
|
3312
|
+
image_path,
|
|
3313
|
+
region,
|
|
3314
|
+
confidence,
|
|
3315
|
+
delay,
|
|
3316
|
+
mouse_button,
|
|
3317
|
+
max_attempts,
|
|
3318
|
+
False,
|
|
3319
|
+
specific,
|
|
3320
|
+
sendtext,
|
|
3321
|
+
show_overlay,
|
|
3322
|
+
wait_until_found,
|
|
3323
|
+
wait_until_disappears,
|
|
3324
|
+
wait_timeout,
|
|
3325
|
+
occurrence=occurrence,
|
|
3326
|
+
)
|
|
3327
|
+
|
|
3328
|
+
|
|
3329
|
+
def click_at(location, mouse_button="left", delay=0, show_overlay=True, config=None):
|
|
3330
|
+
"""
|
|
3331
|
+
Click on specific coordinates using a temporary VisusBot instance.
|
|
3332
|
+
|
|
3333
|
+
Args:
|
|
3334
|
+
location (tuple): (x, y, width, height) of the location
|
|
3335
|
+
mouse_button (str): Mouse button ("left", "right", "double", "move_to")
|
|
3336
|
+
delay (float): Delay after the click
|
|
3337
|
+
show_overlay (bool): Whether to show the red overlay before the click
|
|
3338
|
+
config (dict, optional): Custom configuration
|
|
3339
|
+
|
|
3340
|
+
Returns:
|
|
3341
|
+
bool: True if clicked successfully
|
|
3342
|
+
"""
|
|
3343
|
+
bot = VisusBot(config)
|
|
3344
|
+
return bot.click_at(location, mouse_button, delay, show_overlay)
|
|
3345
|
+
|
|
3346
|
+
|
|
3347
|
+
def type_text(text, config=None):
|
|
3348
|
+
"""
|
|
3349
|
+
Type text using a temporary VisusBot instance.
|
|
3350
|
+
|
|
3351
|
+
Args:
|
|
3352
|
+
text (str): Text to type (supports special commands)
|
|
3353
|
+
config (dict, optional): Custom configuration
|
|
3354
|
+
|
|
3355
|
+
Returns:
|
|
3356
|
+
bool: True if typed successfully
|
|
3357
|
+
"""
|
|
3358
|
+
bot = VisusBot(config)
|
|
3359
|
+
return bot.type_text(text)
|
|
3360
|
+
|
|
3361
|
+
|
|
3362
|
+
# PUBLIC EXPORTS
|
|
3363
|
+
__all__ = [
|
|
3364
|
+
# Main class
|
|
3365
|
+
"VisusBot",
|
|
3366
|
+
"BotVision", # backward-compatibility alias
|
|
3367
|
+
# Task execution functions
|
|
3368
|
+
"execute_tasks",
|
|
3369
|
+
"click_images", # Full compatibility with legacy code
|
|
3370
|
+
"run_automation", # Compatibility with if __name__ == '__main__'
|
|
3371
|
+
# Basic convenience functions
|
|
3372
|
+
"find_text",
|
|
3373
|
+
"click_text",
|
|
3374
|
+
"find_image",
|
|
3375
|
+
"click_image",
|
|
3376
|
+
"click_at",
|
|
3377
|
+
"type_text",
|
|
3378
|
+
# NEW FEATURES - Convenience functions for relative images
|
|
3379
|
+
"find_relative_image",
|
|
3380
|
+
"click_relative_image",
|
|
3381
|
+
# NEW FEATURES - Convenience functions for coordinates
|
|
3382
|
+
"click_coordinates",
|
|
3383
|
+
# NEW FEATURES - Convenience functions for the keyboard
|
|
3384
|
+
"type_text_standalone",
|
|
3385
|
+
"keyboard_command_standalone",
|
|
3386
|
+
"get_available_keyboard_commands",
|
|
3387
|
+
# NEW FEATURE - Text extraction
|
|
3388
|
+
"extract_text_from_region_standalone",
|
|
3389
|
+
# Individual backtrack functions
|
|
3390
|
+
"start_individual_session",
|
|
3391
|
+
"run_individual_tasks_with_backtrack",
|
|
3392
|
+
# Core classes (for advanced use)
|
|
3393
|
+
"TaskExecutor",
|
|
3394
|
+
"TaskResult",
|
|
3395
|
+
"OCREngine",
|
|
3396
|
+
"OCRResult",
|
|
3397
|
+
"ImageProcessor",
|
|
3398
|
+
"VisualOverlay",
|
|
3399
|
+
"RelativeImageDetector", # NEW
|
|
3400
|
+
"KeyboardCommander", # NEW
|
|
3401
|
+
# Utilities
|
|
3402
|
+
"BotVisionConfig",
|
|
3403
|
+
"get_default_config",
|
|
3404
|
+
"clean_text",
|
|
3405
|
+
"limpar_texto",
|
|
3406
|
+
"matches_filter",
|
|
3407
|
+
"retry",
|
|
3408
|
+
"Session",
|
|
3409
|
+
# Standalone processing functions (compatibility)
|
|
3410
|
+
"preprocess_image_for_ocr",
|
|
3411
|
+
"find_text_with_multiple_preprocessing",
|
|
3412
|
+
"show_overlay",
|
|
3413
|
+
"locate_image_with_retry",
|
|
3414
|
+
# Files / clipboard / screen / process / window / image map / mouse (BotCity parity)
|
|
3415
|
+
"wait_for_file",
|
|
3416
|
+
"wait_for_file_stable",
|
|
3417
|
+
"copy_to_clipboard",
|
|
3418
|
+
"paste",
|
|
3419
|
+
"get_clipboard",
|
|
3420
|
+
"capture_screen",
|
|
3421
|
+
"save_screenshot",
|
|
3422
|
+
"take_screenshot",
|
|
3423
|
+
"get_display_size",
|
|
3424
|
+
"get_monitors",
|
|
3425
|
+
"Monitor",
|
|
3426
|
+
"wait_for_stable_screen",
|
|
3427
|
+
"capture_element",
|
|
3428
|
+
"get_window",
|
|
3429
|
+
"smart_find",
|
|
3430
|
+
"smart_click",
|
|
3431
|
+
"launch",
|
|
3432
|
+
"find_process",
|
|
3433
|
+
"is_process_running",
|
|
3434
|
+
"kill_process",
|
|
3435
|
+
"focus_window",
|
|
3436
|
+
"maximize_window",
|
|
3437
|
+
"minimize_window",
|
|
3438
|
+
"close_window",
|
|
3439
|
+
"ensure_window",
|
|
3440
|
+
"list_windows",
|
|
3441
|
+
"scroll",
|
|
3442
|
+
"drag",
|
|
3443
|
+
"triple_click",
|
|
3444
|
+
"move_random",
|
|
3445
|
+
"mouse_down",
|
|
3446
|
+
"mouse_up",
|
|
3447
|
+
"ImageMap",
|
|
3448
|
+
"find_all",
|
|
3449
|
+
"find_multiple",
|
|
3450
|
+
"Backend",
|
|
3451
|
+
"NativeError",
|
|
3452
|
+
# Exceptions
|
|
3453
|
+
"BotVisionError",
|
|
3454
|
+
"ImageNotFoundError",
|
|
3455
|
+
"TextNotFoundError",
|
|
3456
|
+
"TaskExecutionError",
|
|
3457
|
+
"ImageProcessingError",
|
|
3458
|
+
"OCRProcessingError",
|
|
3459
|
+
"TesseractNotFoundError",
|
|
3460
|
+
"InvalidRegionError",
|
|
3461
|
+
"ConfigurationError",
|
|
3462
|
+
"NotSupportedOnThisOS",
|
|
3463
|
+
# Library information
|
|
3464
|
+
"__version__",
|
|
3465
|
+
"__author__",
|
|
3466
|
+
"__email__",
|
|
3467
|
+
"__license__",
|
|
3468
|
+
]
|