visus-desktop 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. visus/desktop/__init__.py +3468 -0
  2. visus/desktop/cli/__init__.py +103 -0
  3. visus/desktop/cli/_common.py +190 -0
  4. visus/desktop/cli/_output.py +56 -0
  5. visus/desktop/cli/commands/__init__.py +1 -0
  6. visus/desktop/cli/commands/capture.py +70 -0
  7. visus/desktop/cli/commands/clipboard.py +32 -0
  8. visus/desktop/cli/commands/config.py +143 -0
  9. visus/desktop/cli/commands/doctor.py +30 -0
  10. visus/desktop/cli/commands/image.py +218 -0
  11. visus/desktop/cli/commands/keyboard.py +52 -0
  12. visus/desktop/cli/commands/mouse.py +118 -0
  13. visus/desktop/cli/commands/native.py +190 -0
  14. visus/desktop/cli/commands/process.py +93 -0
  15. visus/desktop/cli/commands/project.py +86 -0
  16. visus/desktop/cli/commands/screen.py +93 -0
  17. visus/desktop/cli/commands/smart.py +86 -0
  18. visus/desktop/cli/commands/text.py +292 -0
  19. visus/desktop/cli/commands/window.py +146 -0
  20. visus/desktop/cli/repl.py +44 -0
  21. visus/desktop/core/__init__.py +52 -0
  22. visus/desktop/core/capabilities.py +538 -0
  23. visus/desktop/core/elements.py +218 -0
  24. visus/desktop/core/image_map.py +102 -0
  25. visus/desktop/core/image_processing.py +276 -0
  26. visus/desktop/core/keyboard_commands.py +424 -0
  27. visus/desktop/core/keyboard_extra.py +517 -0
  28. visus/desktop/core/locate.py +328 -0
  29. visus/desktop/core/locators.py +324 -0
  30. visus/desktop/core/mouse_extra.py +117 -0
  31. visus/desktop/core/native.py +252 -0
  32. visus/desktop/core/native_rpa.py +482 -0
  33. visus/desktop/core/ocr_engine.py +974 -0
  34. visus/desktop/core/overlay.py +600 -0
  35. visus/desktop/core/overlay_win32.py +403 -0
  36. visus/desktop/core/process.py +151 -0
  37. visus/desktop/core/relative_image.py +232 -0
  38. visus/desktop/core/report.py +607 -0
  39. visus/desktop/core/screen.py +369 -0
  40. visus/desktop/core/session.py +275 -0
  41. visus/desktop/core/smart.py +198 -0
  42. visus/desktop/core/task_executor.py +1802 -0
  43. visus/desktop/core/virtual_mouse.py +548 -0
  44. visus/desktop/core/wait_manager.py +317 -0
  45. visus/desktop/core/window.py +430 -0
  46. visus/desktop/core/window_linux.py +83 -0
  47. visus/desktop/doctor.py +198 -0
  48. visus/desktop/exceptions.py +88 -0
  49. visus/desktop/mcp_server.py +2111 -0
  50. visus/desktop/platform.py +51 -0
  51. visus/desktop/py.typed +0 -0
  52. visus/desktop/resources/cli.md +503 -0
  53. visus/desktop/resources/mcp.md +485 -0
  54. visus/desktop/utils/__init__.py +32 -0
  55. visus/desktop/utils/clipboard.py +68 -0
  56. visus/desktop/utils/config.py +585 -0
  57. visus/desktop/utils/files.py +89 -0
  58. visus/desktop/utils/retry.py +96 -0
  59. visus/desktop/utils/text_filters.py +170 -0
  60. visus_desktop-0.1.0.dist-info/METADATA +1106 -0
  61. visus_desktop-0.1.0.dist-info/RECORD +66 -0
  62. visus_desktop-0.1.0.dist-info/WHEEL +5 -0
  63. visus_desktop-0.1.0.dist-info/entry_points.txt +7 -0
  64. visus_desktop-0.1.0.dist-info/licenses/LICENSE +21 -0
  65. visus_desktop-0.1.0.dist-info/top_level.txt +2 -0
  66. visus_pytest/__init__.py +202 -0
@@ -0,0 +1,3468 @@
1
+ r"""
2
+ Visus Desktop — Advanced GUI Automation (OCR + image + native control).
3
+
4
+ A Python library to automate Windows desktop apps by four interchangeable
5
+ techniques: native accessibility control (no pixels), template-image matching,
6
+ OCR text matching, and absolute coordinates. Pick a technique, or let the
7
+ smart cascade try them in order. Everything is also exposed as an MCP server
8
+ (for AI agents) and a terminal CLI (`visus`).
9
+
10
+ ═══════════════════════════════════════════════════════════════════════════════
11
+ ⚠️ REQUIREMENT: TESSERACT OCR (only for the OCR features)
12
+ ═══════════════════════════════════════════════════════════════════════════════
13
+ 1. Download: https://github.com/UB-Mannheim/tesseract/wiki
14
+ 2. Install to: C:\\Program Files\\Tesseract-OCR\\
15
+ 3. Add it to the system PATH
16
+ 4. Test: tesseract --version
17
+ (Not needed if you set config['ocr_backend']='rapidocr' or only use
18
+ native / image / coordinate targeting. Run `visus doctor` to check.)
19
+
20
+ ═══════════════════════════════════════════════════════════════════════════════
21
+ 📦 INSTALLATION
22
+ ═══════════════════════════════════════════════════════════════════════════════
23
+ pip install visus-desktop # core
24
+ pip install "visus-desktop[native]" # + pywinauto (native RPA, Win)
25
+ pip install "visus-desktop[mcp]" # + MCP server (visus-mcp)
26
+ pip install "visus-desktop[rapidocr]" # + ONNX OCR backend (no Tesseract)
27
+
28
+ ═══════════════════════════════════════════════════════════════════════════════
29
+ 🚀 QUICKSTART
30
+ ═══════════════════════════════════════════════════════════════════════════════
31
+ >>> from visus.desktop import VisusBot, ensure_window
32
+ >>> ensure_window(process="notepad.exe") # open/restore/focus first
33
+ >>> bot = VisusBot()
34
+ >>> bot.click_text("Login") # OCR-find text and click
35
+ >>> bot.type_text("admin") # type into focused field
36
+ >>> bot.click_image("save_button.png") # template-match and click
37
+
38
+ ═══════════════════════════════════════════════════════════════════════════════
39
+ 🧱 THE API — every public name, by category
40
+ ═══════════════════════════════════════════════════════════════════════════════
41
+
42
+ ▶ MAIN CLASS
43
+ VisusBot(config=None)
44
+ The central automation object. Key methods:
45
+ .find_text(text, region=None, confidence_threshold=75, occurrence=1)
46
+ .click_text(text, region=None, mouse_button="left", sendtext=None,
47
+ occurrence=1, backtrack=False)
48
+ .find_image(image_path, region=None, confidence=0.9)
49
+ .click_image(image_path, region=None, confidence=0.9,
50
+ mouse_button="left", backtrack=False)
51
+ .click_at((x, y, w, h), mouse_button="left")
52
+ .type_text(text, interval=0.05)
53
+ .keyboard_command(command) # "Enter", "Ctrl+S", ... (case-sensitive)
54
+ .extract_text_from_region(region, filter_type="both", ...)
55
+ .execute_tasks(tasks, resume=False)
56
+ .start_task_session() / .end_task_session() # manual backtrack scope
57
+ BotVision — backward-compatibility alias for VisusBot.
58
+
59
+ ▶ TASK-LIST EXECUTION (run a whole flow, with retry/backtrack)
60
+ execute_tasks(tasks, resume=False)
61
+ tasks is a list of dicts. Shapes:
62
+ {"image": "btn.png", "region": (x,y,w,h), "confidence": 0.9,
63
+ "mouse_button": "left", "backtrack": True}
64
+ {"text": "Login", "region": (x,y,w,h), "char_type": "letters",
65
+ "sendtext": "user", "occurrence": 1}
66
+ {"type": "type_text", "text": "hello", "interval": 0.05}
67
+ {"type": "keyboard_command", "command": "Escape"} # case-sensitive token
68
+ {"type": "click", "x": 500, "y": 300, "mouse_button": "right"}
69
+ {"type": "relative_image", "anchor_image": "icon.png",
70
+ "target_image": "ok.png", "max_distance": 200}
71
+ {"type": "extract_text", "region": (x,y,w,h)}
72
+ Any task: backtrack, wait_until_found, wait_until_disappears, delay.
73
+ click_images(...) — legacy task-list helper (compatibility).
74
+ run_automation(...) — legacy `__main__` entry helper (compatibility).
75
+
76
+ ▶ FIND / CLICK CONVENIENCE (one-shot, builds a temp bot internally)
77
+ find_text(text, region=None, confidence_threshold=75, occurrence=1) -> box|None
78
+ click_text(text, region=None, mouse_button="left", sendtext=None, ...) -> bool
79
+ find_image(image_path, region=None, confidence=0.9) -> box|None
80
+ click_image(image_path, region=None, confidence=0.9, ...) -> bool
81
+ click_at(x, y, mouse_button="left") -> bool # absolute coords (last resort)
82
+ type_text(text) -> bool
83
+
84
+ ▶ RELATIVE-IMAGE (find a target near a unique anchor)
85
+ find_relative_image(anchor_image, target_image, max_distance=200, ...) -> box|None
86
+ click_relative_image(anchor_image, target_image, ...) -> bool
87
+
88
+ ▶ COORDINATES / KEYBOARD
89
+ click_coordinates(x, y, mouse_button="left") -> bool
90
+ type_text_standalone(text) -> bool
91
+ keyboard_command_standalone(command) -> bool # e.g. "Ctrl+S", "Escape"
92
+ get_available_keyboard_commands() -> list[str] # the valid, case-sensitive tokens
93
+
94
+ ▶ TEXT EXTRACTION (OCR a region, no click)
95
+ extract_text_from_region_standalone(region, filter_type="both",
96
+ return_full_data=False, confidence_threshold=50) -> list|dict
97
+
98
+ ▶ SMART CASCADE & COORDINATE-FREE CAPTURE (recommended for robustness)
99
+ smart_find(query=None, image=None, coordinates=None, window=None,
100
+ control_type=None, order=None, ...) -> {found, box, method, tried}
101
+ smart_click(...) -> {clicked, method, tried}
102
+ One call tries native → image → OCR → coordinates (configurable order)
103
+ and reports which won. The simplest robust "find/click this".
104
+ capture_element(label, auto=None, text=None, native=None, region=None,
105
+ window=None, click=False, elements_dir="./visus-elements")
106
+ -> {found, located_by, box, template_path, annotated_path, clicked}
107
+ Locate an element (Win32 menu → OCR → UIA), save a reusable PNG
108
+ template + an annotated screenshot, optionally click it. The
109
+ coordinate-free way to build a click-by-image flow step by step.
110
+
111
+ ▶ NATIVE RPA (no pixels — accessibility tree; needs the [native] extra)
112
+ Import from visus.desktop.core.native_rpa:
113
+ native_dump_tree(window=None, pid=None, max_depth=3) -> [ {depth,
114
+ control_type, title, auto_id, rect} ] # "see" the UI structure
115
+ native_click(window/pid, title=, control_type=, auto_id=, method="auto")
116
+ native_set_text(text, ...selector...) ; native_get_text(...) -> str|None
117
+ native_get_state(...) -> {exists,visible,enabled,checked,value,rect}
118
+ native_wait(..., state="ready", timeout=10)
119
+ Immune to theme/DPI/resolution/language; empty on non-accessible UIs
120
+ (Java Swing without Access Bridge) — fall back to OCR/image there.
121
+
122
+ ▶ WINDOWS / PROCESSES (fast win32; target the right app first)
123
+ ensure_window(title=None, process=None, show=True, foreground=True)
124
+ -> {found, running, hwnd, pid, title, minimized, maximized,
125
+ foreground, rect, shown, matches}
126
+ ★ CALL THIS FIRST. Is the app open / minimized / where? — and brings
127
+ it to the front. Prefer process= (avoids title false-positives).
128
+ list_windows(title=None, process=None) -> [ {hwnd,title,pid,minimized,...} ]
129
+ get_window(title_substring) -> {title,pid,hwnd,x,y,width,height}|None
130
+ focus_window / maximize_window / minimize_window / close_window(title) -> bool
131
+ launch(command, args=None) -> pid ; find_process(name) -> [Process]
132
+ is_process_running(name) -> bool ; kill_process(pid) -> bool
133
+
134
+ ▶ SCREEN / MONITORS
135
+ take_screenshot(region=None, monitor=None) # PIL.Image
136
+ save_screenshot(path, region=None, monitor=None) -> {path,width,height}
137
+ capture_screen(region=None, monitor=None) ; get_display_size() -> (w,h)
138
+ get_monitors() -> [Monitor] ; Monitor(index,x,y,width,height,is_primary)
139
+ wait_for_stable_screen(region=None, stable_for=0.5, timeout=15) -> bool
140
+
141
+ ▶ FILES / CLIPBOARD
142
+ wait_for_file(path, timeout=10) -> bool
143
+ wait_for_file_stable(path, timeout=30, stable_for=1.0) -> bool # downloads
144
+ copy_to_clipboard(text) / paste() / get_clipboard() -> str
145
+
146
+ ▶ MOUSE (low-level gestures)
147
+ scroll(clicks) drag(x1,y1,x2,y2) triple_click(x,y)
148
+ move_random(...) mouse_down(button) mouse_up(button)
149
+
150
+ ▶ IMAGE MAP / MULTI-MATCH
151
+ ImageMap(mapping) # named templates -> reuse by key
152
+ find_all(image, region=None) ; find_multiple(images)
153
+
154
+ ▶ SESSIONS / REPORTS
155
+ Session.start_recording(zip) # context manager around execute_tasks
156
+ Session.replay(zip) -> [events] # reads events; does NOT re-run
157
+ Session.write_report(zip, html) # styled HTML report
158
+
159
+ ▶ CORE CLASSES (advanced)
160
+ TaskExecutor, TaskResult, OCREngine, OCRResult, ImageProcessor,
161
+ VisualOverlay, RelativeImageDetector, KeyboardCommander, Backend
162
+
163
+ ▶ CONFIG / UTILITIES
164
+ BotVisionConfig(config_dict=None) get_default_config() -> dict
165
+ Hover the class for the full reference grouped by topic. Pass a
166
+ partial dict; everything else uses the defaults. Common keys:
167
+ confidence_threshold, retry_attempts, default_delay,
168
+ show_overlay / overlay_color / overlay_duration / overlay_width,
169
+ default_mouse_button, use_virtual_mouse, click_duration,
170
+ movement_duration, matcher ("cv2"|"pyscreeze"),
171
+ ocr_backend ("tesseract"|"rapidocr"), ocr_languages,
172
+ wait_timeout, screenshot_delay, log_level, log_format
173
+ ("text"|"json"), artifacts_dir, screenshot_on_failure.
174
+ ``log_level`` and ``log_format`` take effect immediately on
175
+ construction; mutate at runtime via cfg.config[key] = value.
176
+ retry(...) # decorator: re-run a callable on transient failure
177
+ clean_text(s) / limpar_texto(s) matches_filter(text, filter_type)
178
+ preprocess_image_for_ocr(img) find_text_with_multiple_preprocessing(...)
179
+ show_overlay(region) locate_image_with_retry(image, ...)
180
+
181
+ ▶ EXCEPTIONS
182
+ BotVisionError (base), ImageNotFoundError, TextNotFoundError,
183
+ TaskExecutionError, ImageProcessingError, OCRProcessingError,
184
+ TesseractNotFoundError, InvalidRegionError, ConfigurationError,
185
+ NotSupportedOnThisOS, NativeError
186
+
187
+ ═══════════════════════════════════════════════════════════════════════════════
188
+ 🤖 USING THE MCP SERVER (drive everything from an AI agent)
189
+ ═══════════════════════════════════════════════════════════════════════════════
190
+ Install + run:
191
+ pip install "visus-desktop[mcp]"
192
+ visus-mcp # stdio transport (what hosts expect)
193
+
194
+ Wire into Claude Code / Desktop / Cursor (settings.json):
195
+ { "mcpServers": { "visus-desktop": { "command": "visus-mcp" } } }
196
+ Restart the host; 57 tools become available to the agent.
197
+
198
+ Tool groups (each tool's docstring tells the agent when/how to use it):
199
+ • Targeting: ensure_window ★, list_windows, get_window
200
+ • Find/click: find_text, find_all_text, click_text, find_image,
201
+ click_image, click_at, hybrid_find
202
+ • Smart/capture: smart_find, smart_click, capture_element
203
+ • Native RPA: native_dump_tree, native_click, native_set_text,
204
+ native_get_text, native_get_state, native_wait
205
+ • Keyboard: type_text, keyboard_command
206
+ • Screen: take_screenshot, save_screenshot, get_monitors,
207
+ wait_for_stable_screen
208
+ • Files/clip: wait_for_file, wait_for_file_stable, get_clipboard,
209
+ copy_to_clipboard, extract_text_from_region
210
+ • Process/win: launch, find_process, kill_process, focus_window,
211
+ maximize_window, minimize_window, close_window
212
+ • Task lists: execute_tasks
213
+ • Sessions: start_recording, stop_recording, replay_session, render_report
214
+ • Diagnostics: doctor, configure
215
+
216
+ Recommended agent flow:
217
+ ensure_window(process="app.exe") # open/focus, get rect
218
+ native_dump_tree(window="App") # if accessible -> act natively
219
+ capture_element(label="save", auto="Save", window="App", click=True)
220
+ # or smart_click(query="Save", window="App")
221
+ start_recording("run.zip"); execute_tasks([...]); stop_recording()
222
+ render_report("run.zip", "run.html")
223
+ Full reference: docs/mcp.md. Agent playbooks: skills/.
224
+
225
+ ═══════════════════════════════════════════════════════════════════════════════
226
+ ⌨️ TERMINAL CLI — "Visus CLI Pro" (the `visus` command)
227
+ ═══════════════════════════════════════════════════════════════════════════════
228
+ Mirrors the MCP surface in the shell (great for scripts/CI). --json on any:
229
+ visus window ensure --process notepad.exe
230
+ visus smart click --query "Login" --dry-run
231
+ visus capture save --auto "Save" --click
232
+ visus native dump-tree --window "Notepad" --json
233
+ visus repl # interactive shell
234
+ visus doctor # environment health check
235
+ Full reference: docs/cli.md.
236
+
237
+ ═══════════════════════════════════════════════════════════════════════════════
238
+ 💡 TIPS & TROUBLESHOOTING
239
+ ═══════════════════════════════════════════════════════════════════════════════
240
+ • Always ensure_window(process=) first; scope find/click with region=.
241
+ • backtrack=True for fragile multi-step flows; retry() for arbitrary calls.
242
+ • Element not found → lower confidence / widen region / re-capture template.
243
+ • Inaccurate OCR → narrow region, filter_type="letters", or try rapidocr.
244
+ • Wrong window clicked → match by process, not title; pass the window rect.
245
+ • Run `visus doctor` (or the doctor MCP tool) to diagnose the environment.
246
+ """
247
+
248
+ import functools
249
+ import logging
250
+ import time
251
+
252
+ from .core.capabilities import CapabilitiesMixin
253
+ from .core.elements import capture_element
254
+ from .core.image_map import ImageMap
255
+ from .core.image_processing import ImageProcessor, preprocess_image_for_ocr
256
+ from .core.keyboard_commands import KeyboardCommander # NEW FEATURE
257
+ from .core.keyboard_extra import KeyboardMixin
258
+ from .core.locate import find_all, find_multiple
259
+ from .core.locators import get_window
260
+ from .core.mouse_extra import (
261
+ drag,
262
+ mouse_down,
263
+ mouse_up,
264
+ move_random,
265
+ scroll,
266
+ triple_click,
267
+ )
268
+ from .core.native import Backend, NativeError
269
+ from .core.ocr_engine import OCREngine, OCRResult, find_text_with_multiple_preprocessing
270
+ from .core.overlay import VisualOverlay, show_overlay
271
+ from .core.process import find_process, is_process_running, kill_process, launch
272
+ from .core.relative_image import RelativeImageDetector # NEW FEATURE
273
+ from .core.screen import (
274
+ Monitor,
275
+ capture_screen,
276
+ get_display_size,
277
+ get_monitors,
278
+ save_screenshot,
279
+ take_screenshot,
280
+ wait_for_stable_screen,
281
+ )
282
+ from .core.session import Session
283
+ from .core.smart import smart_click, smart_find
284
+
285
+ # Import the main classes
286
+ from .core.task_executor import TaskExecutor, TaskResult
287
+ from .core.window import (
288
+ close_window,
289
+ ensure_window,
290
+ focus_window,
291
+ list_windows,
292
+ maximize_window,
293
+ minimize_window,
294
+ )
295
+ from .exceptions import (
296
+ BotVisionError,
297
+ ConfigurationError,
298
+ ImageNotFoundError,
299
+ ImageProcessingError,
300
+ InvalidRegionError,
301
+ NotSupportedOnThisOS,
302
+ OCRProcessingError,
303
+ TaskExecutionError,
304
+ TesseractNotFoundError,
305
+ TextNotFoundError,
306
+ )
307
+ from .utils.clipboard import copy as copy_to_clipboard
308
+ from .utils.clipboard import get_clipboard, paste
309
+ from .utils.config import BotVisionConfig, get_default_config
310
+ from .utils.files import wait_for_file, wait_for_file_stable
311
+ from .utils.retry import retry
312
+ from .utils.text_filters import clean_text, limpar_texto, matches_filter
313
+
314
+ # Library version
315
+ __version__ = "0.1.0"
316
+ __author__ = "Matheus Pereira"
317
+ __email__ = "matheuszwilkdev@gmail.com"
318
+ __license__ = "MIT"
319
+
320
+ # Default logging configuration
321
+ logger = logging.getLogger(__name__)
322
+
323
+
324
+ # Keys we copy from the wrapped method's kwargs into the recorded event
325
+ # so reports show the relevant context per call (the executor's own
326
+ # ``_emit_task_event`` picks these up via ``_task_type`` / ``_task_target``).
327
+ _RECORDED_KWARGS = (
328
+ "text", "image", "image_path", "anchor_image", "target_image",
329
+ "command", "region", "x", "y", "confidence", "confidence_threshold",
330
+ "mouse_button", "occurrence", "specific", "max_distance", "filter_type",
331
+ )
332
+
333
+
334
+ def _records_in_session(type_field):
335
+ """Decorator: emit a structured ``task_event`` for the wrapped call.
336
+
337
+ Applied to public ``VisusBot`` action methods (``click_text``,
338
+ ``click_image``, ``type_text``, …). When the call happens inside a
339
+ ``with bot.start_task_session()`` scope, the result is forwarded to
340
+ :meth:`VisusBot._emit_direct_call_event` so a parallel
341
+ :class:`Session.start_recording` zip contains the same event line +
342
+ per-step / failure screenshot it would have captured via
343
+ :meth:`execute_tasks`. Outside a session the wrapper is transparent.
344
+ """
345
+ def decorator(method):
346
+ @functools.wraps(method)
347
+ def wrapper(self, *args, **kwargs):
348
+ if not getattr(self, "task_session_active", False):
349
+ return method(self, *args, **kwargs)
350
+
351
+ started = time.monotonic()
352
+ try:
353
+ result = method(self, *args, **kwargs)
354
+ except Exception as exc:
355
+ self._emit_direct_call_event(
356
+ _build_event_task(type_field, method, args, kwargs),
357
+ success=False,
358
+ started_at=started,
359
+ error=str(exc),
360
+ )
361
+ raise
362
+
363
+ success = bool(result) if isinstance(result, bool) else True
364
+ self._emit_direct_call_event(
365
+ _build_event_task(type_field, method, args, kwargs),
366
+ success=success,
367
+ started_at=started,
368
+ error=None if success else f"{method.__name__} returned falsy",
369
+ )
370
+ return result
371
+ return wrapper
372
+ return decorator
373
+
374
+
375
+ def _build_event_task(type_field, method, args, kwargs):
376
+ """Construct the task dict consumed by ``_emit_task_event``.
377
+
378
+ Maps the first positional arg to a sensible task field based on
379
+ ``type_field`` (so ``click_text("Login")`` ends up as
380
+ ``{"type": "text", "text": "Login"}``), and folds in any recognized
381
+ kwargs that add useful context to the event.
382
+ """
383
+ task = {"type": type_field}
384
+ if args:
385
+ primary_key = {
386
+ "text": "text",
387
+ "image": "image",
388
+ "click": "x", # click_at / click_coordinates -- args[0] is x or location
389
+ "relative_image": "anchor_image",
390
+ "type_text": "text",
391
+ "keyboard_command": "command",
392
+ }.get(type_field)
393
+ if primary_key:
394
+ task[primary_key] = args[0]
395
+ else:
396
+ task["arg"] = args[0]
397
+ for key in _RECORDED_KWARGS:
398
+ if key in kwargs and key not in task:
399
+ task[key] = kwargs[key]
400
+ return task
401
+
402
+
403
+ class VisusBot(CapabilitiesMixin, KeyboardMixin):
404
+ """High-level GUI automation: native control, image, OCR, coordinates.
405
+
406
+ ``VisusBot`` is the single entry point users typically import. It bundles
407
+ the smart-cascade finder (native → image → OCR → coordinates), a
408
+ keyboard/mouse driver (physical or virtual), an OCR engine, and a
409
+ click-through visual overlay for debugging. Every public method is also
410
+ exposed via the ``visus`` CLI and the ``visus-mcp`` MCP server.
411
+
412
+ Built-in capabilities (see method docstrings for full signatures):
413
+
414
+ **Finding things on screen**
415
+
416
+ * :meth:`find_text` / :meth:`click_text` -- OCR-based text targeting
417
+ with confidence threshold, character-class filter, and per-region
418
+ search.
419
+ * :meth:`find_image` / :meth:`click_image` -- Template matching with
420
+ ``cv2.matchTemplate`` (falls back to pyscreeze); supports
421
+ scale-variation search when ``specific=False``.
422
+ * :meth:`find_relative_image` -- Find a target image near an anchor
423
+ (good for picking the right row in a repeating grid).
424
+ * :meth:`hybrid_find` -- Try the smart cascade in order until one
425
+ method locates the target.
426
+
427
+ **Acting on things**
428
+
429
+ * :meth:`click_at` -- Click absolute (x, y, w, h) coordinates.
430
+ * :meth:`type_text` -- Type a string into the focused field.
431
+ * Mouse extras from :class:`CapabilitiesMixin`: :meth:`drag`,
432
+ :meth:`triple_click`, :meth:`scroll`, :meth:`move_random`.
433
+ * Keyboard from :class:`KeyboardMixin`: hotkeys, key presses,
434
+ special-sequence parsing (``{ctrl}a{del}{tab}``).
435
+
436
+ **Robustness primitives**
437
+
438
+ * Backtrack: pass ``backtrack=True`` to retry failed steps and
439
+ re-walk the recent action history (see
440
+ :meth:`start_task_session`).
441
+ * Waits: ``wait_until_found`` / ``wait_until_disappears`` /
442
+ ``wait_timeout`` on find/click methods.
443
+ * Failure artifacts: when ``screenshot_on_failure=True`` is set in
444
+ the config, every uncaught failure dumps a PNG + JSON blob into
445
+ ``artifacts_dir``.
446
+
447
+ **Overlay (visual feedback)**
448
+
449
+ * :meth:`configure_overlay`, :meth:`get_overlay_config`,
450
+ :attr:`show_overlay`. On Windows the overlay is a layered
451
+ click-through window (``WS_EX_TRANSPARENT``), so the click that
452
+ follows lands on the host below regardless of tech (native,
453
+ Java, Electron, browsers).
454
+
455
+ Examples:
456
+ Defaults are usually fine; install Tesseract and go::
457
+
458
+ from visus.desktop import VisusBot, ensure_window
459
+ ensure_window(process="notepad.exe")
460
+ bot = VisusBot()
461
+ bot.click_text("File")
462
+ bot.type_text("hello world")
463
+
464
+ Tune via a dict (delegated to :class:`BotVisionConfig`)::
465
+
466
+ bot = VisusBot(config={
467
+ "confidence_threshold": 80.0,
468
+ "overlay_color": "green",
469
+ "matcher": "cv2",
470
+ })
471
+
472
+ Or build the config object yourself for finer control::
473
+
474
+ from visus.desktop import BotVisionConfig
475
+ cfg = BotVisionConfig({"show_overlay": False})
476
+ bot = VisusBot(config=cfg)
477
+ """
478
+
479
+ def __init__(self, config=None):
480
+ """Initialize Visus Desktop.
481
+
482
+ Wires the executor, OCR engine, image processor, and overlay
483
+ master toggle. Construction is cheap; Tesseract is auto-detected only
484
+ when ``ocr_backend == "tesseract"`` (the default backend is now
485
+ ``rapidocr``, which uses bundled models and skips detection).
486
+
487
+ Args:
488
+ config: Custom settings. Three forms are accepted:
489
+
490
+ * ``None`` (default) -- use the library defaults.
491
+ * ``dict`` -- a mapping of any subset of the keys
492
+ documented on :class:`BotVisionConfig` (hover that class
493
+ for the full reference of every key, type, and default).
494
+ Common ones:
495
+
496
+ - ``confidence_threshold`` (float, ``75.0``): OCR cutoff.
497
+ - ``retry_attempts`` (int, ``3``): default retries.
498
+ - ``default_delay`` (float, ``1.0``): pause between
499
+ actions (s).
500
+ - ``show_overlay`` (bool, ``True``): master toggle for
501
+ the click-through visual overlay.
502
+ - ``overlay_color`` (str, ``"red"``) /
503
+ ``overlay_duration`` (int ms, ``1000``) /
504
+ ``overlay_width`` (int px, ``4``): overlay look.
505
+ - ``default_mouse_button`` (str, ``"left"``): one of
506
+ ``"left"``, ``"right"``, ``"double"``, ``"move_to"``.
507
+ - ``use_virtual_mouse`` (bool, ``False``): click without
508
+ moving the physical cursor (Windows, PostMessage).
509
+ - ``matcher`` (str, ``"cv2"``): ``"cv2"`` or
510
+ ``"pyscreeze"``.
511
+ - ``ocr_backend`` (str, ``"rapidocr"``): ``"rapidocr"``
512
+ (default) or ``"tesseract"``.
513
+ - ``ocr_languages`` (list[str], ``["eng"]``).
514
+ - ``wait_timeout`` (int s, ``30``): default wait timeout.
515
+ - ``log_level`` (str, ``"INFO"``) /
516
+ ``log_format`` (str, ``"text"`` or ``"json"``).
517
+ - ``artifacts_dir`` (str, ``"./visus-artifacts"``) /
518
+ ``screenshot_on_failure`` (bool, ``True``).
519
+
520
+ * :class:`BotVisionConfig` -- a pre-built config object,
521
+ used as-is. Use this when you want to reuse the same
522
+ config across several bots or when you have already
523
+ validated it.
524
+
525
+ Examples:
526
+ >>> bot = VisusBot() # defaults
527
+ >>> bot = VisusBot(config={"overlay_color": "green"})
528
+ >>> from visus.desktop import BotVisionConfig
529
+ >>> cfg = BotVisionConfig({"show_overlay": False})
530
+ >>> bot = VisusBot(config=cfg)
531
+ """
532
+ if isinstance(config, dict):
533
+ self.config = BotVisionConfig(config)
534
+ elif isinstance(config, BotVisionConfig):
535
+ self.config = config
536
+ else:
537
+ self.config = get_default_config()
538
+
539
+ # Bind the hybrid finder lazily so that monkeypatches on
540
+ # ``self.hybrid_find`` (e.g. in tests) take effect for hybrid
541
+ # task dispatch in the executor.
542
+ self.executor = TaskExecutor(
543
+ self.config,
544
+ hybrid_finder=lambda name, **kw: self.hybrid_find(name, **kw),
545
+ )
546
+ self.ocr_engine = OCREngine(self.config)
547
+ self.image_processor = ImageProcessor()
548
+
549
+ # Overlay master toggle - accessible via the ``show_overlay`` property.
550
+ self._show_overlay = self.config.get("show_overlay", True)
551
+
552
+ # Backtrack system for individual methods
553
+ self.individual_task_history = [] # History of executed methods
554
+ self.backtrack_enabled_globally = False # Whether it is in backtrack mode
555
+ self.current_task_index = -1 # Index of the current task
556
+ self.max_backtrack_attempts = 2 # Maximum backtrack attempts per task
557
+ self.task_session_active = False # Whether it is in an individual task session
558
+
559
+ def _add_to_task_history(self, method_name, args, kwargs):
560
+ """Add a method to the individual task history."""
561
+ task_info = {
562
+ "method": method_name,
563
+ "args": args,
564
+ "kwargs": kwargs,
565
+ "index": len(self.individual_task_history),
566
+ "success": None,
567
+ "backtrack_attempts": 0,
568
+ "timestamp": time.time(),
569
+ }
570
+ self.individual_task_history.append(task_info)
571
+ self.current_task_index = len(self.individual_task_history) - 1
572
+ return self.current_task_index
573
+
574
+ def start_task_session(self):
575
+ """Open a manual task session.
576
+
577
+ While a session is active, direct calls like :meth:`click_text` and
578
+ :meth:`click_image` emit the same structured ``task_event`` as
579
+ :meth:`execute_tasks` so they show up in a
580
+ :class:`~visus.desktop.core.session.Session` recording (events +
581
+ per-step screenshots in the zip). Also enables the backtrack
582
+ history for retries between methods.
583
+ """
584
+ import uuid
585
+
586
+ self.task_session_active = True
587
+ self.individual_task_history = []
588
+ self.current_task_index = -1
589
+ # Tag every event emitted during this session with a stable run id
590
+ # so the resulting events.jsonl groups cleanly in reports.
591
+ self.executor.current_run_id = uuid.uuid4().hex[:8]
592
+ logger.debug(
593
+ "Task session started (run_id=%s) -- direct calls will be recorded",
594
+ self.executor.current_run_id,
595
+ )
596
+
597
+ def end_task_session(self):
598
+ """Close the manual task session opened by :meth:`start_task_session`."""
599
+ self.task_session_active = False
600
+ successful_tasks = sum(
601
+ 1 for task in self.individual_task_history if task.get("success", False)
602
+ )
603
+ total_tasks = len(self.individual_task_history)
604
+ logger.debug(
605
+ "Task session ended: %d/%d successful tasks (run_id=%s)",
606
+ successful_tasks, total_tasks, self.executor.current_run_id,
607
+ )
608
+ self.executor.current_run_id = None
609
+ return successful_tasks, total_tasks
610
+
611
+ def _emit_direct_call_event(
612
+ self, task_for_event, success, started_at, location=None, error=None
613
+ ):
614
+ """Emit a structured ``task_event`` for a direct method call.
615
+
616
+ No-op unless a task session is open (so direct calls outside a
617
+ session keep their original "no zip noise" behavior). Mirrors the
618
+ shape of :meth:`TaskExecutor._emit_task_event`, including the
619
+ success / failure screenshot decision driven by
620
+ ``screenshot_steps`` and ``screenshot_on_failure``.
621
+ """
622
+ if not getattr(self, "task_session_active", False):
623
+ return
624
+ if self.config.get("log_format", "text") != "json":
625
+ return
626
+
627
+ task_index = len(self.individual_task_history)
628
+ self.individual_task_history.append({
629
+ "method": task_for_event.get("type") or task_for_event.get("text") or "direct",
630
+ "success": bool(success),
631
+ })
632
+
633
+ target = (
634
+ task_for_event.get("text")
635
+ or task_for_event.get("image")
636
+ or task_for_event.get("type", "direct_call")
637
+ )
638
+ # If the caller (decorator) didn't pass a location, fall back to
639
+ # the one stashed by ``TaskExecutor._perform_action`` -- that's
640
+ # what direct ``click_text`` / ``click_image`` calls landed on,
641
+ # so the screenshot can be annotated even though the public
642
+ # method only returned bool.
643
+ effective_location = location or getattr(
644
+ self.executor, "last_action_location", None
645
+ )
646
+ result = TaskResult(task_index, bool(success), str(target), effective_location)
647
+ result.attempts = 1
648
+ result.duration_ms = int((time.monotonic() - started_at) * 1000)
649
+ result.error = error
650
+ if success and self.config.get("screenshot_steps"):
651
+ result.step_screenshot = self.executor._capture_step_screenshot(
652
+ task_index, location=effective_location
653
+ )
654
+ elif not success and self.config.get("screenshot_on_failure", True):
655
+ result.failure_screenshot = self.executor._save_failure_screenshot(
656
+ task_index, location=effective_location
657
+ )
658
+ self.executor._emit_task_event(result, task_for_event)
659
+
660
+ # Overlay control properties
661
+ @property
662
+ def show_overlay(self):
663
+ """Control whether the visual overlay is displayed before clicks."""
664
+ return self._show_overlay
665
+
666
+ @show_overlay.setter
667
+ def show_overlay(self, value):
668
+ """Set whether the visual overlay is displayed before clicks."""
669
+ self._show_overlay = bool(value)
670
+ # Also update the configuration
671
+ self.config.config["show_overlay"] = self._show_overlay
672
+
673
+ def configure_overlay(self, enabled=None, color=None, duration=None, width=None):
674
+ """
675
+ Conveniently configure the overlay parameters.
676
+
677
+ Args:
678
+ enabled (bool, optional): Whether the overlay is enabled
679
+ color (str, optional): Overlay color - options: 'red', 'blue', 'green', 'yellow',
680
+ 'purple', 'orange', 'cyan', 'magenta', 'white', 'black'
681
+ duration (int, optional): Duration in milliseconds (500-5000 recommended)
682
+ width (int, optional): Overlay line width (1-10 recommended)
683
+
684
+ Examples:
685
+ >>> bot = VisusBot()
686
+ >>> bot.configure_overlay(enabled=True, color="blue", duration=2000)
687
+ >>> bot.configure_overlay(color="green", width=6)
688
+
689
+ Raises:
690
+ ValueError: If an invalid color is provided
691
+ """
692
+ valid_colors = [
693
+ "red",
694
+ "blue",
695
+ "green",
696
+ "yellow",
697
+ "purple",
698
+ "orange",
699
+ "cyan",
700
+ "magenta",
701
+ "white",
702
+ "black",
703
+ ]
704
+
705
+ if enabled is not None:
706
+ self.show_overlay = enabled
707
+
708
+ if color is not None:
709
+ if color.lower() not in [c.lower() for c in valid_colors]:
710
+ raise ValueError(
711
+ f"Invalid color '{color}'. Available colors: {', '.join(valid_colors)}"
712
+ )
713
+ self.config.config["overlay_color"] = color.lower()
714
+
715
+ if duration is not None:
716
+ if not isinstance(duration, (int, float)) or duration <= 0:
717
+ raise ValueError(
718
+ f"Duration must be a positive number, received: {duration}"
719
+ )
720
+ if duration > 10000:
721
+ logger.warning(
722
+ f"Duration too high ({duration}ms). Consider using less than 5000ms."
723
+ )
724
+ self.config.config["overlay_duration"] = int(duration)
725
+
726
+ if width is not None:
727
+ if not isinstance(width, (int, float)) or width <= 0:
728
+ raise ValueError(f"Width must be a positive number, received: {width}")
729
+ if width > 15:
730
+ logger.warning(
731
+ f"Width too high ({width}). Consider using less than 10."
732
+ )
733
+ self.config.config["overlay_width"] = int(width)
734
+
735
+ def get_overlay_config(self):
736
+ """
737
+ Return the current overlay configuration.
738
+
739
+ Returns:
740
+ dict: Overlay settings
741
+ """
742
+ return {
743
+ "enabled": self.show_overlay,
744
+ "color": self.config.config.get("overlay_color", "red"),
745
+ "duration": self.config.config.get("overlay_duration", 1000),
746
+ "width": self.config.config.get("overlay_width", 4),
747
+ }
748
+
749
+ @staticmethod
750
+ def get_available_overlay_colors():
751
+ """
752
+ Return the list of available overlay colors.
753
+
754
+ Returns:
755
+ list: List of available colors
756
+
757
+ Examples:
758
+ >>> colors = VisusBot.get_available_overlay_colors()
759
+ >>> print("Available colors:", colors)
760
+ """
761
+ return [
762
+ "red",
763
+ "blue",
764
+ "green",
765
+ "yellow",
766
+ "purple",
767
+ "orange",
768
+ "cyan",
769
+ "magenta",
770
+ "white",
771
+ "black",
772
+ ]
773
+
774
+ def test_overlay_colors(self, duration=1500):
775
+ """
776
+ Test all available overlay colors.
777
+
778
+ Args:
779
+ duration (int): Duration of each color in milliseconds
780
+
781
+ Examples:
782
+ >>> bot = VisusBot()
783
+ >>> bot.test_overlay_colors() # Shows each color for 1.5 seconds
784
+ """
785
+ from .core.overlay import VisualOverlay
786
+
787
+ colors = self.get_available_overlay_colors()
788
+ test_region = (400, 300, 200, 100) # Center of the screen
789
+
790
+ print("🎨 Testing overlay colors...")
791
+ print("📍 Look at the center of the screen!")
792
+
793
+ for i, color in enumerate(colors, 1):
794
+ print(f" {i}. Color: {color.upper()}")
795
+ overlay = VisualOverlay(color=color, width=6, duration=duration)
796
+ overlay.show(test_region, blocking=True)
797
+
798
+ print("✅ Color test complete!")
799
+
800
+ def _execute_with_individual_backtrack(
801
+ self, method_name, method_func, *args, **kwargs
802
+ ):
803
+ """
804
+ Execute an individual method with backtrack capability.
805
+
806
+ Args:
807
+ method_name (str): Name of the method ('click_image', 'click_text', etc.)
808
+ method_func (callable): The method function to execute
809
+ *args: Positional arguments
810
+ **kwargs: Keyword arguments
811
+
812
+ Returns:
813
+ bool: Result of the execution
814
+ """
815
+ backtrack = kwargs.get("backtrack", False)
816
+
817
+ # If backtrack is not enabled or there is no active session, execute normally
818
+ if not backtrack or not self.task_session_active:
819
+ if backtrack and not self.task_session_active:
820
+ # Auto-start a session if backtrack=True but no session is active
821
+ self.start_task_session()
822
+
823
+ # Remove backtrack from kwargs to avoid infinite recursion
824
+ execution_kwargs = kwargs.copy()
825
+ execution_kwargs.pop("backtrack", None)
826
+ return method_func(*args, **execution_kwargs)
827
+
828
+ # Add to the history
829
+ task_index = self._add_to_task_history(method_name, args, kwargs)
830
+
831
+ logger.info(
832
+ f"🎯 Running {method_name} (task {task_index + 1}) with backtrack enabled"
833
+ )
834
+
835
+ # Remove backtrack from kwargs to avoid infinite recursion
836
+ execution_kwargs = kwargs.copy()
837
+ execution_kwargs.pop("backtrack", None)
838
+
839
+ # Try to execute the method
840
+ success = method_func(*args, **execution_kwargs)
841
+
842
+ # Update the result in the history
843
+ self.individual_task_history[task_index]["success"] = success
844
+
845
+ if success:
846
+ logger.info(
847
+ f"✓ {method_name} (task {task_index + 1}) executed successfully!"
848
+ )
849
+ return True
850
+ else:
851
+ # Failed - check whether a backtrack is possible
852
+ if task_index > 0: # There are previous tasks
853
+ current_task = self.individual_task_history[task_index]
854
+
855
+ if current_task["backtrack_attempts"] < self.max_backtrack_attempts:
856
+ current_task["backtrack_attempts"] += 1
857
+
858
+ # Execute the previous task
859
+ prev_task = self.individual_task_history[task_index - 1]
860
+ logger.info(
861
+ f"🔄 BACKTRACK: {method_name} (task {task_index + 1}) failed. "
862
+ f"Re-running the previous task ({prev_task['method']}) and trying again..."
863
+ )
864
+
865
+ # Re-run the previous task
866
+ prev_method_name = prev_task["method"]
867
+ prev_method_func = getattr(self, f"_{prev_method_name}_internal")
868
+ prev_kwargs = prev_task["kwargs"].copy()
869
+ prev_kwargs.pop(
870
+ "backtrack", None
871
+ ) # Remove backtrack to avoid recursion
872
+
873
+ prev_success = prev_method_func(*prev_task["args"], **prev_kwargs)
874
+ self.individual_task_history[task_index - 1]["success"] = (
875
+ prev_success
876
+ )
877
+
878
+ if prev_success:
879
+ logger.info(
880
+ f"✓ Previous task ({prev_task['method']}) re-run successfully"
881
+ )
882
+
883
+ # Wait a bit and try the current task again
884
+ time.sleep(0.5)
885
+ logger.info(
886
+ f"🔄 Retrying {method_name} (task {task_index + 1}) after backtrack..."
887
+ )
888
+
889
+ retry_success = method_func(*args, **execution_kwargs)
890
+ self.individual_task_history[task_index]["success"] = (
891
+ retry_success
892
+ )
893
+
894
+ if retry_success:
895
+ logger.info(
896
+ f"✓ {method_name} (task {task_index + 1}) succeeded after backtrack!"
897
+ )
898
+ return True
899
+ else:
900
+ logger.warning(
901
+ f"✗ {method_name} (task {task_index + 1}) still failed after backtrack"
902
+ )
903
+ else:
904
+ logger.warning(
905
+ f"✗ Previous task ({prev_task['method']}) also failed on re-run"
906
+ )
907
+ else:
908
+ logger.warning(
909
+ f"✗ {method_name} (task {task_index + 1}) failed after {self.max_backtrack_attempts} backtrack attempts"
910
+ )
911
+ else:
912
+ logger.warning(
913
+ f"✗ {method_name} (task {task_index + 1}) failed but it is the first task (no backtrack possible)"
914
+ )
915
+
916
+ return False
917
+
918
+ # ------------------------------------------------------------------
919
+ # Hybrid targeting (Phase 3 / T1)
920
+ # ------------------------------------------------------------------
921
+
922
+ _IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".bmp", ".gif", ".tiff")
923
+
924
+ # Control types queried (in order) before falling back to an
925
+ # unfiltered descendants() walk. Filtering by control_type at the UIA
926
+ # level is far cheaper than pulling the whole accessibility tree, so
927
+ # clickable elements (menu items, buttons, tabs, links) are found fast.
928
+ _NATIVE_CONTROL_TYPES = (
929
+ "MenuItem",
930
+ "Button",
931
+ "TabItem",
932
+ "ListItem",
933
+ "Hyperlink",
934
+ "Text",
935
+ "Edit",
936
+ "CheckBox",
937
+ "RadioButton",
938
+ "ComboBox",
939
+ )
940
+
941
+ def _hybrid_native_lookup(self, name, region=None, native_timeout: float = 5.0):
942
+ """Best-effort native UIA lookup used by :meth:`hybrid_find`.
943
+
944
+ Searches the foreground window's UIA tree (via ``pywinauto.Desktop``)
945
+ for a control whose **title**, **automation_id**, or **control_type**
946
+ contains ``name`` (case-insensitive). Returns an ``(x, y, w, h)`` box
947
+ or ``None`` when not found.
948
+
949
+ Two safeguards keep UIA's notoriously slow tree enumeration from
950
+ hanging the caller:
951
+
952
+ * **control_type filtering** -- cheap ``descendants(control_type=...)``
953
+ queries for common clickable types run first; the expensive
954
+ unfiltered walk is only the last resort.
955
+ * **``native_timeout``** -- the whole lookup runs in a worker thread
956
+ and is abandoned (returns ``None``) if it exceeds the timeout, so
957
+ the hybrid chain falls back to OCR promptly instead of blocking.
958
+
959
+ On non-Windows / missing pywinauto it returns ``None`` cleanly.
960
+ """
961
+ try:
962
+ import pywinauto # type: ignore[import-not-found] # noqa: F401
963
+ except ImportError:
964
+ return None
965
+
966
+ result: dict = {"box": None}
967
+
968
+ def _work() -> None:
969
+ try:
970
+ import pywinauto as _pwa
971
+
972
+ desktop = _pwa.Desktop(backend="uia")
973
+ top = desktop.window(active_only=True)
974
+ if not top.exists(timeout=0.5):
975
+ return
976
+ needle = (name or "").lower()
977
+ if not needle:
978
+ return
979
+
980
+ def _match(descendant):
981
+ try:
982
+ info = descendant.element_info
983
+ title = (info.name or "").lower()
984
+ auto_id = (getattr(info, "automation_id", "") or "").lower()
985
+ ctrl = (getattr(info, "control_type", "") or "").lower()
986
+ except Exception:
987
+ return None
988
+ if needle in title or needle in auto_id or needle in ctrl:
989
+ try:
990
+ r = descendant.rectangle()
991
+ except Exception:
992
+ return None
993
+ return (
994
+ int(r.left),
995
+ int(r.top),
996
+ int(r.right - r.left),
997
+ int(r.bottom - r.top),
998
+ )
999
+ return None
1000
+
1001
+ # Cheap filtered passes first, then the unfiltered fallback.
1002
+ queries = [
1003
+ {"control_type": ct} for ct in self._NATIVE_CONTROL_TYPES
1004
+ ]
1005
+ queries.append({}) # unfiltered last resort
1006
+ for kwargs in queries:
1007
+ try:
1008
+ candidates = top.descendants(**kwargs)
1009
+ except Exception:
1010
+ continue
1011
+ for descendant in candidates:
1012
+ box = _match(descendant)
1013
+ if box is not None:
1014
+ result["box"] = box
1015
+ return
1016
+ except Exception as exc: # any pywinauto-level fault is non-fatal
1017
+ logger.debug("native hybrid lookup failed: %s", exc)
1018
+
1019
+ import threading
1020
+
1021
+ worker = threading.Thread(target=_work, daemon=True)
1022
+ worker.start()
1023
+ worker.join(timeout=native_timeout)
1024
+ if worker.is_alive():
1025
+ logger.debug(
1026
+ "native lookup exceeded %.1fs; giving up (falling back to OCR)",
1027
+ native_timeout,
1028
+ )
1029
+ return None
1030
+ return result["box"]
1031
+
1032
+ def hybrid_find(self, name, region=None):
1033
+ """Find a target by trying native → OCR → image in order.
1034
+
1035
+ Each layer is mockable; the first hit wins. Returns an
1036
+ ``(x, y, w, h)`` tuple or ``None``. The image layer is only
1037
+ attempted when ``name`` looks like an image filename (one of
1038
+ ``.png .jpg .jpeg .bmp .gif .tiff``).
1039
+ """
1040
+ # Layer 1 -- native UI handle (pywinauto).
1041
+ try:
1042
+ box = self._hybrid_native_lookup(name, region=region)
1043
+ except Exception:
1044
+ box = None
1045
+ if box:
1046
+ return box
1047
+
1048
+ # Layer 2 -- OCR.
1049
+ box = self.find_text(name, region=region)
1050
+ if box:
1051
+ return box
1052
+
1053
+ # Layer 3 -- image template (only for paths that look like images).
1054
+ lower = name.lower() if isinstance(name, str) else ""
1055
+ if lower.endswith(self._IMAGE_SUFFIXES):
1056
+ box = self.find_image(name, region=region)
1057
+ if box:
1058
+ return box
1059
+ return None
1060
+
1061
+ def execute_tasks(self, tasks, resume: bool = False):
1062
+ """
1063
+ Execute a list of tasks sequentially.
1064
+
1065
+ Args:
1066
+ tasks (list): List of dictionaries with task settings
1067
+ resume (bool): When True, skip indices already completed in a
1068
+ previous interrupted run (Phase 2 / R3). The checkpoint
1069
+ file ``<artifacts_dir>/checkpoint.json`` is invalidated
1070
+ when the task list changes shape.
1071
+
1072
+ Returns:
1073
+ list: List of TaskResult objects with the result of each task
1074
+
1075
+ Examples:
1076
+ >>> # Basic tasks
1077
+ >>> tasks = [
1078
+ ... {'text': 'Login', 'region': (100, 100, 500, 300)},
1079
+ ... {'image': 'button.png', 'delay': 2}
1080
+ ... ]
1081
+ >>>
1082
+ >>> # Advanced tasks with ALL the new features:
1083
+ >>> advanced_tasks = [
1084
+ ... # 1. OCR text search
1085
+ ... {
1086
+ ... 'text': 'Usuário',
1087
+ ... 'region': (100, 100, 500, 300),
1088
+ ... 'char_type': 'letters',
1089
+ ... 'confidence': 0.8,
1090
+ ... 'occurrence': 1,
1091
+ ... 'backtrack': True,
1092
+ ... 'delay': 1,
1093
+ ... 'sendtext': 'admin{tab}password{enter}'
1094
+ ... },
1095
+ ... # 2. Relative image search (NEW!)
1096
+ ... {
1097
+ ... 'type': 'relative_image',
1098
+ ... 'anchor_image': 'warning_icon.png',
1099
+ ... 'target_image': 'ok_button.png',
1100
+ ... 'max_distance': 200,
1101
+ ... 'confidence': 0.9,
1102
+ ... 'target_region': (0, 0, 800, 600),
1103
+ ... 'specific': True,
1104
+ ... 'backtrack': True,
1105
+ ... 'delay': 1
1106
+ ... },
1107
+ ... # 3. Click on specific coordinates (NEW!)
1108
+ ... {
1109
+ ... 'type': 'click',
1110
+ ... 'x': 100,
1111
+ ... 'y': 200,
1112
+ ... 'mouse_button': 'right',
1113
+ ... 'delay': 0.5,
1114
+ ... 'backtrack': False
1115
+ ... },
1116
+ ... # 4. Text typing (NEW!)
1117
+ ... {
1118
+ ... 'type': 'type_text',
1119
+ ... 'text': 'Hello World!',
1120
+ ... 'interval': 0.05,
1121
+ ... 'delay': 1
1122
+ ... },
1123
+ ... # 5. Keyboard command (NEW!)
1124
+ ... {
1125
+ ... 'type': 'keyboard_command',
1126
+ ... 'command': 'Ctrl+S',
1127
+ ... 'delay': 1
1128
+ ... }
1129
+ ... ]
1130
+ >>>
1131
+ >>> results = bot.execute_tasks(advanced_tasks)
1132
+ """
1133
+ return self.executor.execute_tasks(tasks, resume=resume)
1134
+
1135
+ def find_text(
1136
+ self,
1137
+ text,
1138
+ region=None,
1139
+ filter_type="both",
1140
+ confidence_threshold=75.0,
1141
+ occurrence=1,
1142
+ max_attempts=3,
1143
+ backtrack=False,
1144
+ wait_until_found=False,
1145
+ wait_until_disappears=False,
1146
+ wait_timeout=None,
1147
+ word_box=False,
1148
+ ):
1149
+ """
1150
+ Find text on screen using advanced OCR with all features.
1151
+
1152
+ Args:
1153
+ text (str): Text to find
1154
+ region (tuple, optional): (x, y, width, height) of the search region
1155
+ filter_type (str): Filter type ("numbers", "letters", "both") - equivalent to char_type
1156
+ confidence_threshold (float): Minimum confidence threshold
1157
+ occurrence (int): Which occurrence to look for (1 = first, 2 = second, etc.)
1158
+ max_attempts (int): Maximum number of attempts if backtrack=True
1159
+ backtrack (bool): Whether to try multiple times with adjustments
1160
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
1161
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
1162
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
1163
+
1164
+ Returns:
1165
+ tuple: Coordinates (x, y, width, height) where the text was found, or None
1166
+
1167
+ Examples:
1168
+ >>> location = bot.find_text("Confirmar", region=(0, 0, 800, 600), backtrack=True)
1169
+ >>> if location:
1170
+ ... print(f"Text found at: {location}")
1171
+ """
1172
+ import time
1173
+
1174
+ # If there are wait parameters, build a task and use the executor with wait
1175
+ if wait_until_found or wait_until_disappears:
1176
+ temp_task = {
1177
+ "text": text,
1178
+ "region": region,
1179
+ "char_type": filter_type,
1180
+ "confidence_threshold": confidence_threshold,
1181
+ "occurrence": occurrence,
1182
+ "wait_until_found": wait_until_found,
1183
+ "wait_until_disappears": wait_until_disappears,
1184
+ "wait_timeout": wait_timeout,
1185
+ }
1186
+
1187
+ # Use the executor's wait-aware method
1188
+ if wait_until_found:
1189
+ return self.executor._find_text_with_wait(temp_task, 0)
1190
+ elif wait_until_disappears:
1191
+ return self.executor._find_text_with_wait(temp_task, 0)
1192
+
1193
+ # Original behavior without wait
1194
+ attempts = 0
1195
+ while attempts < max_attempts:
1196
+ if region is None:
1197
+ # If no region was specified, capture the whole screen
1198
+ import pyautogui
1199
+
1200
+ screen = pyautogui.screenshot()
1201
+ region = (0, 0, screen.width, screen.height)
1202
+ region_img = screen
1203
+ else:
1204
+ region_img = self.executor._capture_region(region)
1205
+
1206
+ # Whether the captured image already spans the whole region
1207
+ # (so box coords are already absolute) or needs offsetting.
1208
+ _full = region == (0, 0, region_img.width, region_img.height)
1209
+ _ox, _oy = (0, 0) if _full else (region[0], region[1])
1210
+
1211
+ # Backend-agnostic OCR narrative on the ``visus`` logger, so it
1212
+ # shows up in a session recording regardless of which OCR engine
1213
+ # is active (the engines' own loggers are noisy and filtered out).
1214
+ backend = self.config.get("ocr_backend", "rapidocr")
1215
+ logger.info(
1216
+ "OCR search: '%s' (occurrence=%d, filter=%s, backend=%s) in region %s",
1217
+ text, occurrence, filter_type, backend, region,
1218
+ )
1219
+
1220
+ if occurrence > 1 or word_box:
1221
+ # ``ocr_engine.find_text`` early-exits with only the first
1222
+ # high-confidence match (and only at line granularity), so it
1223
+ # can't serve the Nth occurrence nor a word-level box. Enumerate
1224
+ # ALL matches (deduped, reading order) and pick the Nth; pass
1225
+ # ``word_box`` so the box is tightened to the word itself.
1226
+ matches = self.ocr_engine.find_all_text(
1227
+ region_img, text, filter_type, confidence_threshold,
1228
+ word_box=word_box,
1229
+ )
1230
+ logger.info(
1231
+ "OCR found %d occurrence(s) of '%s'", len(matches), text
1232
+ )
1233
+ if len(matches) >= occurrence:
1234
+ conf, bx = matches[occurrence - 1]
1235
+ box = (_ox + bx[0], _oy + bx[1], bx[2], bx[3])
1236
+ logger.info(
1237
+ "Selected occurrence %d of '%s' at %s (confidence %.1f%%)",
1238
+ occurrence, text, box, conf,
1239
+ )
1240
+ return box
1241
+ logger.warning(
1242
+ "Occurrence %d of '%s' not found (%d match(es))",
1243
+ occurrence, text, len(matches),
1244
+ )
1245
+ else:
1246
+ found_boxes, scores, _ = self.ocr_engine.find_text(
1247
+ region_img, text, filter_type, confidence_threshold
1248
+ )
1249
+ if found_boxes:
1250
+ bx = found_boxes[0]
1251
+ box = (_ox + bx[0], _oy + bx[1], bx[2], bx[3])
1252
+ conf = scores[0] if scores else 0.0
1253
+ logger.info(
1254
+ "Found '%s' at %s (confidence %.1f%%)", text, box, conf
1255
+ )
1256
+ return box
1257
+ logger.warning("Text '%s' not found in region %s", text, region)
1258
+
1259
+ # If nothing was found and backtrack is enabled, try again
1260
+ if backtrack and attempts < max_attempts - 1:
1261
+ attempts += 1
1262
+ logger.info(f"Attempt {attempts}/{max_attempts} to find '{text}'")
1263
+
1264
+ # Adjust parameters for the next attempt
1265
+ confidence_threshold = max(60.0, confidence_threshold - 5.0)
1266
+ time.sleep(0.5)
1267
+ else:
1268
+ break
1269
+
1270
+ return None
1271
+
1272
+ def find_all_text(
1273
+ self,
1274
+ text,
1275
+ region=None,
1276
+ filter_type="both",
1277
+ confidence_threshold=None,
1278
+ word_box=False,
1279
+ ):
1280
+ """List every occurrence of ``text`` visible on screen (or in ``region``).
1281
+
1282
+ Unlike :meth:`find_text` (which stops at the first high-confidence
1283
+ hit), this scans the OCR output exhaustively and returns every
1284
+ distinct match. Use it when the same string appears multiple
1285
+ times and you need to disambiguate by position -- e.g. clicking
1286
+ the second ``"OK"`` on a form, or numbering visible rows.
1287
+
1288
+ Args:
1289
+ text (str): Substring to look for (case-insensitive, after
1290
+ ``filter_type`` normalization).
1291
+ region (tuple, optional): ``(x, y, w, h)`` search area in
1292
+ screen coordinates. ``None`` searches the full screen.
1293
+ filter_type (str): ``"letters"`` / ``"numbers"`` / ``"both"``.
1294
+ confidence_threshold (float, optional): Minimum OCR confidence
1295
+ (0-100). Defaults to ``config.confidence_threshold``.
1296
+ word_box (bool): When True, narrow each box from the whole OCR
1297
+ text line to just ``text`` itself (handy to locate a word
1298
+ inside a long run such as a URL). Default False keeps the full
1299
+ line box. On RapidOCR this uses the engine's native per-word
1300
+ boxes (``return_word_box``), falling back to a character-offset
1301
+ estimate if unavailable; a no-op on Tesseract, whose boxes are
1302
+ already word-level.
1303
+
1304
+ Returns:
1305
+ list[tuple]: ``[(x, y, w, h, confidence), ...]`` in screen
1306
+ coordinates, sorted in reading order (top-to-bottom,
1307
+ left-to-right). Spatially overlapping detections collapse
1308
+ to a single entry that keeps the best confidence.
1309
+
1310
+ Examples:
1311
+ >>> matches = bot.find_all_text("Confirm")
1312
+ >>> for i, (x, y, w, h, conf) in enumerate(matches, start=1):
1313
+ ... print(f" #{i} at ({x}, {y}) confidence={conf:.1f}%")
1314
+ """
1315
+ import pyautogui
1316
+
1317
+ if region is None:
1318
+ screen = pyautogui.screenshot()
1319
+ region = (0, 0, screen.width, screen.height)
1320
+ region_img = screen
1321
+ else:
1322
+ region_img = self.executor._capture_region(region)
1323
+
1324
+ threshold = (
1325
+ confidence_threshold
1326
+ if confidence_threshold is not None
1327
+ else self.config.confidence_threshold
1328
+ )
1329
+ matches = self.ocr_engine.find_all_text(
1330
+ region_img, text, filter_type, threshold, word_box=word_box
1331
+ )
1332
+ # Box is relative to the captured region; convert to screen coordinates.
1333
+ return [
1334
+ (region[0] + box[0], region[1] + box[1], box[2], box[3], conf)
1335
+ for conf, box in matches
1336
+ ]
1337
+
1338
+ def show_text_occurrences(
1339
+ self,
1340
+ text,
1341
+ region=None,
1342
+ filter_type="both",
1343
+ confidence_threshold=None,
1344
+ color="red",
1345
+ duration_ms=3000,
1346
+ width=3,
1347
+ word_box=False,
1348
+ ):
1349
+ """Overlay every occurrence of ``text`` on screen, numbered ``1, 2, 3...``.
1350
+
1351
+ Calls :meth:`find_all_text` and draws a click-through Win32
1352
+ overlay around each match with the position number above the
1353
+ top-left corner. Overlays fade after ``duration_ms`` and do not
1354
+ intercept clicks. On non-Windows the overlays are skipped but
1355
+ the match list is still returned.
1356
+
1357
+ Args:
1358
+ text (str): Substring to look for.
1359
+ region (tuple, optional): Search region.
1360
+ filter_type (str): OCR char filter.
1361
+ confidence_threshold (float, optional): Minimum confidence.
1362
+ color (str): Overlay color.
1363
+ duration_ms (int): How long each overlay stays on screen.
1364
+ width (int): Outline thickness in pixels.
1365
+ word_box (bool): Narrow each box to just ``text`` instead of the
1366
+ whole OCR line (see :meth:`find_all_text`). Default False.
1367
+
1368
+ Returns:
1369
+ list[tuple]: Same shape as :meth:`find_all_text`.
1370
+
1371
+ Examples:
1372
+ >>> matches = bot.show_text_occurrences("Login", duration_ms=4000)
1373
+ >>> print(f"Found {len(matches)} occurrence(s)")
1374
+ """
1375
+ from .core.overlay_win32 import show_click_through_overlay
1376
+
1377
+ matches = self.find_all_text(
1378
+ text, region, filter_type, confidence_threshold, word_box=word_box
1379
+ )
1380
+ for i, (x, y, w, h, _) in enumerate(matches, start=1):
1381
+ show_click_through_overlay(
1382
+ (x, y, w, h),
1383
+ color=color,
1384
+ width=width,
1385
+ duration_ms=duration_ms,
1386
+ label=str(i),
1387
+ )
1388
+ logger.info(
1389
+ f"show_text_occurrences('{text}'): {len(matches)} match(es)"
1390
+ )
1391
+ return matches
1392
+
1393
+ @_records_in_session("text")
1394
+ def click_text(
1395
+ self,
1396
+ text,
1397
+ region=None,
1398
+ filter_type="both",
1399
+ delay=0,
1400
+ mouse_button="left",
1401
+ occurrence=1,
1402
+ backtrack=False,
1403
+ max_attempts=3,
1404
+ sendtext=None,
1405
+ confidence_threshold=None,
1406
+ show_overlay=None,
1407
+ wait_until_found=False,
1408
+ wait_until_disappears=False,
1409
+ wait_timeout=None,
1410
+ word_box=False,
1411
+ ):
1412
+ """
1413
+ Find and click on text with advanced features.
1414
+
1415
+ Args:
1416
+ text (str): Text to click on
1417
+ region (tuple, optional): Search region (x, y, width, height)
1418
+ filter_type (str): Filter type ("numbers", "letters", "both")
1419
+ delay (float): Delay after the click in seconds
1420
+ mouse_button (str): Mouse click type:
1421
+ • "left" - Simple left click (default)
1422
+ • "right" - Right click
1423
+ • "double" or "double left" - Double left click
1424
+ • "move_to" - Just moves the mouse without clicking
1425
+ occurrence (int): Which occurrence to click (1=first)
1426
+ backtrack (bool): Whether to use real backtracking between individual methods
1427
+ max_attempts (int): Maximum number of attempts
1428
+ sendtext (str, optional): Text to type after the click
1429
+ confidence_threshold (float, optional): Custom confidence threshold
1430
+ show_overlay (bool, optional): Whether to show the red overlay before the click
1431
+ If None, uses the global configuration
1432
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
1433
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
1434
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
1435
+
1436
+ Returns:
1437
+ bool: True if found and clicked, False otherwise
1438
+ """
1439
+ # Use the global configuration if not specified
1440
+ if show_overlay is None:
1441
+ show_overlay = self.config.show_overlay
1442
+
1443
+ # If backtrack is True, use the individual backtrack system
1444
+ if backtrack:
1445
+ return self._execute_with_individual_backtrack(
1446
+ "click_text",
1447
+ self._click_text_internal,
1448
+ text,
1449
+ region,
1450
+ filter_type,
1451
+ delay,
1452
+ mouse_button,
1453
+ occurrence,
1454
+ max_attempts,
1455
+ sendtext,
1456
+ confidence_threshold,
1457
+ show_overlay,
1458
+ wait_until_found,
1459
+ wait_until_disappears,
1460
+ wait_timeout,
1461
+ word_box,
1462
+ backtrack=backtrack,
1463
+ )
1464
+
1465
+ # Original behavior without backtrack
1466
+ return self._click_text_internal(
1467
+ text,
1468
+ region,
1469
+ filter_type,
1470
+ delay,
1471
+ mouse_button,
1472
+ occurrence,
1473
+ max_attempts,
1474
+ sendtext,
1475
+ confidence_threshold,
1476
+ show_overlay,
1477
+ wait_until_found,
1478
+ wait_until_disappears,
1479
+ wait_timeout,
1480
+ word_box,
1481
+ )
1482
+
1483
+ def _click_text_internal(
1484
+ self,
1485
+ text,
1486
+ region=None,
1487
+ filter_type="both",
1488
+ delay=0,
1489
+ mouse_button="left",
1490
+ occurrence=1,
1491
+ max_attempts=3,
1492
+ sendtext=None,
1493
+ confidence_threshold=None,
1494
+ show_overlay=None,
1495
+ wait_until_found=False,
1496
+ wait_until_disappears=False,
1497
+ wait_timeout=None,
1498
+ word_box=False,
1499
+ ):
1500
+ """Internal version of click_text without backtrack (for use by the backtrack system)."""
1501
+ # Use the global configuration if not specified
1502
+ if show_overlay is None:
1503
+ show_overlay = self.config.show_overlay
1504
+ attempts = 0
1505
+ while attempts < max_attempts:
1506
+ attempts += 1
1507
+
1508
+ try:
1509
+ if attempts > 1:
1510
+ logger.info(
1511
+ f"Attempt {attempts}/{max_attempts} to click on '{text}'"
1512
+ )
1513
+
1514
+ # Use the custom threshold if provided, otherwise use the one from the configuration
1515
+ threshold = confidence_threshold or self.config.confidence_threshold
1516
+
1517
+ location = self.find_text(
1518
+ text,
1519
+ region,
1520
+ filter_type,
1521
+ threshold,
1522
+ occurrence,
1523
+ max_attempts=1,
1524
+ backtrack=False,
1525
+ wait_until_found=wait_until_found,
1526
+ wait_until_disappears=wait_until_disappears,
1527
+ wait_timeout=wait_timeout,
1528
+ word_box=word_box,
1529
+ )
1530
+
1531
+ if location:
1532
+ # Build a temporary task to use the executor
1533
+ temp_task = {
1534
+ "mouse_button": mouse_button,
1535
+ "delay": delay,
1536
+ "sendtext": sendtext,
1537
+ "show_overlay": show_overlay,
1538
+ }
1539
+
1540
+ try:
1541
+ self.executor._perform_action(temp_task, location)
1542
+ return True
1543
+ except Exception as e:
1544
+ logger.error(f"Error clicking on text: {e}")
1545
+ if attempts >= max_attempts:
1546
+ return False
1547
+ time.sleep(0.5)
1548
+ else:
1549
+ logger.warning(f"Text '{text}' not found on attempt {attempts}")
1550
+ if attempts < max_attempts:
1551
+ time.sleep(0.5)
1552
+
1553
+ except Exception as e:
1554
+ logger.error(f"Error on attempt {attempts} for '{text}': {e}")
1555
+ if attempts >= max_attempts:
1556
+ return False
1557
+ time.sleep(0.5)
1558
+
1559
+ return False
1560
+
1561
+ def _click_image_internal(
1562
+ self,
1563
+ image_path,
1564
+ region=None,
1565
+ confidence=0.9,
1566
+ delay=0,
1567
+ mouse_button="left",
1568
+ max_attempts=3,
1569
+ specific=True,
1570
+ sendtext=None,
1571
+ show_overlay=None,
1572
+ wait_until_found=False,
1573
+ wait_until_disappears=False,
1574
+ wait_timeout=None,
1575
+ occurrence=1,
1576
+ ):
1577
+ """Internal version of click_image without backtrack (for use by the backtrack system)."""
1578
+ # Use the global configuration if not specified
1579
+ if show_overlay is None:
1580
+ show_overlay = self.config.show_overlay
1581
+
1582
+ # For click_image, wait_until_disappears must wait AFTER the click, not before
1583
+ # So we pass only wait_until_found to find_image
1584
+ location = self.find_image(
1585
+ image_path,
1586
+ region,
1587
+ confidence,
1588
+ max_attempts,
1589
+ False,
1590
+ specific,
1591
+ wait_until_found,
1592
+ False,
1593
+ wait_timeout,
1594
+ occurrence=occurrence,
1595
+ ) # backtrack=False for find_image
1596
+
1597
+ if location:
1598
+ # Build a temporary task to use the executor
1599
+ temp_task = {
1600
+ "image": image_path,
1601
+ "region": region,
1602
+ "confidence": confidence,
1603
+ "specific": specific,
1604
+ "mouse_button": mouse_button,
1605
+ "delay": delay,
1606
+ "sendtext": sendtext,
1607
+ "show_overlay": show_overlay,
1608
+ "wait_until_disappears": wait_until_disappears,
1609
+ "wait_timeout": wait_timeout,
1610
+ }
1611
+
1612
+ try:
1613
+ self.executor._perform_action(temp_task, location)
1614
+ return True
1615
+ except Exception as e:
1616
+ logger.error(f"Error clicking on image: {e}")
1617
+ return False
1618
+
1619
+ return False
1620
+
1621
+ def extract_text_from_region(
1622
+ self,
1623
+ region=None,
1624
+ filter_type="both",
1625
+ confidence_threshold=50.0,
1626
+ return_full_data=False,
1627
+ max_attempts=3,
1628
+ backtrack=False,
1629
+ ):
1630
+ """
1631
+ Extract all text found in a specific region of the screen.
1632
+
1633
+ Args:
1634
+ region (tuple, optional): (x, y, width, height) of the region to extract text from.
1635
+ If None, extracts from the whole screen
1636
+ filter_type (str): Filter type for the text:
1637
+ • "numbers" - Numbers only
1638
+ • "letters" - Letters only
1639
+ • "both" - Numbers and letters (default)
1640
+ confidence_threshold (float): Minimum confidence threshold (0-100)
1641
+ return_full_data (bool): If True, returns full data with coordinates and confidence
1642
+ max_attempts (int): Maximum number of attempts if backtrack=True
1643
+ backtrack (bool): Whether to try multiple times with adjustments
1644
+
1645
+ Returns:
1646
+ If return_full_data=False:
1647
+ list: List of strings with all the text found
1648
+ If return_full_data=True:
1649
+ list: List of dictionaries with:
1650
+ - 'text': Text found
1651
+ - 'confidence': Confidence level (0-100)
1652
+ - 'box': Coordinates (x, y, width, height)
1653
+ - 'absolute_box': Absolute coordinates on screen
1654
+
1655
+ Examples:
1656
+ Simple extraction:
1657
+ >>> texts = bot.extract_text_from_region(region=(100, 100, 400, 300))
1658
+ >>> print("Texts found:", texts)
1659
+
1660
+ Extraction with full data:
1661
+ >>> data = bot.extract_text_from_region(
1662
+ ... region=(100, 100, 400, 300),
1663
+ ... return_full_data=True,
1664
+ ... confidence_threshold=70.0
1665
+ ... )
1666
+ >>> for item in data:
1667
+ ... print(f"'{item['text']}' - Confidence: {item['confidence']:.1f}% - Position: {item['absolute_box']}")
1668
+
1669
+ Numbers only:
1670
+ >>> numbers = bot.extract_text_from_region(
1671
+ ... region=(50, 50, 200, 100),
1672
+ ... filter_type="numbers"
1673
+ ... )
1674
+ """
1675
+ import time
1676
+
1677
+ attempts = 0
1678
+ while attempts < max_attempts:
1679
+ try:
1680
+ if region is None:
1681
+ # If no region was specified, capture the whole screen
1682
+ import pyautogui
1683
+
1684
+ screen = pyautogui.screenshot()
1685
+ region = (0, 0, screen.width, screen.height)
1686
+ region_img = screen
1687
+ else:
1688
+ region_img = self.executor._capture_region(region)
1689
+
1690
+ # Use the OCR engine's extract_all_text method
1691
+ ocr_results = self.ocr_engine.extract_all_text(region_img, filter_type)
1692
+
1693
+ # Filter by confidence
1694
+ filtered_results = [
1695
+ result
1696
+ for result in ocr_results
1697
+ if result.confidence >= confidence_threshold
1698
+ ]
1699
+
1700
+ if not filtered_results and backtrack and attempts < max_attempts - 1:
1701
+ # If nothing was found and backtrack is enabled, adjust parameters
1702
+ attempts += 1
1703
+ logger.info(
1704
+ f"Attempt {attempts}/{max_attempts} for text extraction"
1705
+ )
1706
+ confidence_threshold = max(30.0, confidence_threshold - 10.0)
1707
+ time.sleep(0.5)
1708
+ continue
1709
+
1710
+ if return_full_data:
1711
+ # Return full data with absolute coordinates
1712
+ full_data = []
1713
+ for result in filtered_results:
1714
+ absolute_box = (
1715
+ (
1716
+ region[0] + result.box[0],
1717
+ region[1] + result.box[1],
1718
+ result.box[2],
1719
+ result.box[3],
1720
+ )
1721
+ if region != (0, 0, region_img.width, region_img.height)
1722
+ else result.box
1723
+ )
1724
+
1725
+ full_data.append(
1726
+ {
1727
+ "text": result.text,
1728
+ "confidence": result.confidence,
1729
+ "box": result.box, # Coordinates relative to the region
1730
+ "absolute_box": absolute_box, # Absolute coordinates on screen
1731
+ }
1732
+ )
1733
+
1734
+ return full_data
1735
+ else:
1736
+ # Return only the texts
1737
+ return [result.text for result in filtered_results]
1738
+
1739
+ except Exception as e:
1740
+ logger.error(f"Error on attempt {attempts} for text extraction: {e}")
1741
+ if backtrack and attempts < max_attempts - 1:
1742
+ attempts += 1
1743
+ time.sleep(0.5)
1744
+ else:
1745
+ break
1746
+
1747
+ # If it got here, it failed
1748
+ return [] if not return_full_data else []
1749
+
1750
+ def get_last_extracted_text(self):
1751
+ """
1752
+ Get the last extracted text when used in task lists.
1753
+
1754
+ Returns:
1755
+ list: Last text extraction result, or None if no extraction has been done
1756
+
1757
+ Examples:
1758
+ >>> tasks = [
1759
+ ... {
1760
+ ... 'type': 'extract_text',
1761
+ ... 'region': (100, 100, 400, 300),
1762
+ ... 'filter_type': 'both'
1763
+ ... }
1764
+ ... ]
1765
+ >>> results = bot.execute_tasks(tasks)
1766
+ >>> extracted_texts = bot.get_last_extracted_text()
1767
+ >>> print("Texts found:", extracted_texts)
1768
+ """
1769
+ return getattr(self.executor, "last_extracted_text", None)
1770
+
1771
+ def find_image(
1772
+ self,
1773
+ image_path,
1774
+ region=None,
1775
+ confidence=0.9,
1776
+ max_attempts=3,
1777
+ backtrack=False,
1778
+ specific=True,
1779
+ wait_until_found=False,
1780
+ wait_until_disappears=False,
1781
+ wait_timeout=None,
1782
+ scales=None,
1783
+ occurrence=1,
1784
+ ):
1785
+ """
1786
+ Find an image on screen with all the advanced features.
1787
+
1788
+ Args:
1789
+ image_path (str): Path to the image
1790
+ region (tuple, optional): Search region
1791
+ confidence (float): Confidence level
1792
+ max_attempts (int): Maximum number of attempts if backtrack=True
1793
+ backtrack (bool): Whether to try multiple times with adjustments
1794
+ specific (bool): If True, searches in the region; if False, searches the whole screen + scale variations
1795
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
1796
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
1797
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
1798
+ scales (list, optional): List of scales to try (e.g.: [1.0, 0.95, 1.05])
1799
+ occurrence (int): Which match to return when the template appears more
1800
+ than once, 1-based in reading order (top-to-bottom, left-to-right).
1801
+ ``occurrence > 1`` enumerates every match and picks the Nth.
1802
+
1803
+ Returns:
1804
+ tuple: Coordinates of the image or None
1805
+ """
1806
+ import time
1807
+
1808
+ image_path = self.image_map.resolve(image_path)
1809
+
1810
+ # occurrence > 1: a single-best match (the default find path) can't serve
1811
+ # the 2nd/3rd copy. Enumerate every match and pick the Nth in reading
1812
+ # order (top-to-bottom, left-to-right) -- mirrors click_text(occurrence=N).
1813
+ if occurrence and occurrence > 1:
1814
+ from .core.locate import find_all as _find_all
1815
+
1816
+ # Honor `specific` exactly like the single-match path below: restrict
1817
+ # to `region` only when specific=True. With specific=False (search the
1818
+ # whole screen) a captured template carries the FIRST copy's box as
1819
+ # `region`, so restricting find_all to it would only ever see 1 match
1820
+ # and the 2nd/3rd copies (outside that box) could never be selected.
1821
+ search_region = tuple(region) if (specific and region) else None
1822
+ boxes = _find_all(
1823
+ image_path,
1824
+ region=search_region,
1825
+ confidence=confidence,
1826
+ )
1827
+ boxes = sorted(boxes, key=lambda b: (b[1], b[0]))
1828
+ logger.info(
1829
+ "Image '%s' found %d occurrence(s)", image_path, len(boxes)
1830
+ )
1831
+ if len(boxes) >= occurrence:
1832
+ box = boxes[occurrence - 1]
1833
+ logger.info(
1834
+ "Selected image occurrence %d at %s", occurrence, box
1835
+ )
1836
+ return box
1837
+ logger.warning(
1838
+ "Image occurrence %d of '%s' not found (%d match(es))",
1839
+ occurrence, image_path, len(boxes),
1840
+ )
1841
+ return None
1842
+
1843
+ if scales is None:
1844
+ scales = [1.0, 0.95, 1.05] if not specific else [1.0]
1845
+
1846
+ # If there are wait parameters, build a task and use the executor with wait
1847
+ if wait_until_found or wait_until_disappears:
1848
+ temp_task = {
1849
+ "image": image_path,
1850
+ "region": region,
1851
+ "confidence": confidence,
1852
+ "specific": specific,
1853
+ "wait_until_found": wait_until_found,
1854
+ "wait_until_disappears": wait_until_disappears,
1855
+ "wait_timeout": wait_timeout,
1856
+ }
1857
+
1858
+ # Use the executor's wait-aware method
1859
+ return self.executor._find_image_with_wait(temp_task, 0)
1860
+
1861
+ # Original behavior without wait
1862
+ attempts = 0
1863
+ while attempts < max_attempts:
1864
+ try:
1865
+ # Try with different scales if not specific
1866
+ for scale in scales:
1867
+ try:
1868
+ if scale != 1.0 and not specific:
1869
+ # Resize the reference image
1870
+ import os
1871
+
1872
+ from PIL import Image
1873
+
1874
+ ref_img = Image.open(image_path)
1875
+ new_width = int(ref_img.width * scale)
1876
+ new_height = int(ref_img.height * scale)
1877
+ scaled_img = ref_img.resize((new_width, new_height))
1878
+
1879
+ # Save temporarily
1880
+ temp_path = f"temp_scaled_{scale}_{attempts}.png"
1881
+ scaled_img.save(temp_path)
1882
+
1883
+ # NEW LOGIC: specific controls where to search
1884
+ if specific and region:
1885
+ # If specific AND there is a region, search in the region
1886
+ location = self.executor._locate_image_with_retry(
1887
+ temp_path, region, confidence
1888
+ )
1889
+ else:
1890
+ # If not specific OR no region, search the whole screen
1891
+ location = self.executor._locate_image_with_retry(
1892
+ temp_path, None, confidence
1893
+ )
1894
+
1895
+ # Remove the temporary file
1896
+ if os.path.exists(temp_path):
1897
+ os.remove(temp_path)
1898
+ else:
1899
+ # NEW LOGIC: specific controls where to search
1900
+ if specific and region:
1901
+ # If specific AND there is a region, search in the region
1902
+ logger.info(
1903
+ f"Searching for image in region {region} (specific=True)"
1904
+ )
1905
+ location = self.executor._locate_image_with_retry(
1906
+ image_path, region, confidence
1907
+ )
1908
+ else:
1909
+ # If not specific OR no region, search the whole screen
1910
+ logger.info(
1911
+ "Searching for image across the whole screen (specific=False)"
1912
+ )
1913
+ location = self.executor._locate_image_with_retry(
1914
+ image_path, None, confidence
1915
+ )
1916
+
1917
+ if location:
1918
+ return location
1919
+
1920
+ except Exception as e:
1921
+ logger.debug(f"Error searching for image at scale {scale}: {e}")
1922
+ continue
1923
+ # If nothing was found and backtrack is enabled, adjust parameters
1924
+ if backtrack and attempts < max_attempts - 1:
1925
+ attempts += 1
1926
+ logger.info(f"Attempt {attempts}/{max_attempts} to find the image")
1927
+
1928
+ # Gradually lower the confidence
1929
+ confidence = max(0.7, confidence - 0.05)
1930
+ time.sleep(0.5)
1931
+ else:
1932
+ break
1933
+
1934
+ except Exception as e:
1935
+ logger.error(f"Error searching for image: {e}")
1936
+ if not backtrack:
1937
+ break
1938
+
1939
+ return None
1940
+
1941
+ @_records_in_session("image")
1942
+ def click_image(
1943
+ self,
1944
+ image_path,
1945
+ region=None,
1946
+ confidence=0.9,
1947
+ delay=0,
1948
+ mouse_button="left",
1949
+ max_attempts=3,
1950
+ backtrack=False,
1951
+ specific=True,
1952
+ sendtext=None,
1953
+ show_overlay=None,
1954
+ wait_until_found=False,
1955
+ wait_until_disappears=False,
1956
+ wait_timeout=None,
1957
+ occurrence=1,
1958
+ ):
1959
+ """
1960
+ Find and click on an image with all the advanced features.
1961
+
1962
+ Args:
1963
+ image_path (str): Path to the image
1964
+ region (tuple, optional): Search region (x, y, width, height)
1965
+ confidence (float): Confidence level (0.0-1.0)
1966
+ delay (float): Delay after the click in seconds
1967
+ mouse_button (str): Mouse click type:
1968
+ • "left" - Simple left click (default)
1969
+ • "right" - Right click
1970
+ • "double" or "double left" - Double left click
1971
+ • "move_to" - Just moves the mouse without clicking
1972
+ max_attempts (int): Maximum number of attempts if backtrack=True
1973
+ backtrack (bool): Whether to use real backtracking between individual methods
1974
+ specific (bool): If True, exact search; if False, allows variations
1975
+ sendtext (str, optional): Text to type after the click
1976
+ show_overlay (bool, optional): Whether to show the red overlay before the click
1977
+ If None, uses the global configuration
1978
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
1979
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
1980
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
1981
+
1982
+ Returns:
1983
+ bool: True if found and clicked, False otherwise
1984
+ """
1985
+ # Use the global configuration if not specified
1986
+ if show_overlay is None:
1987
+ show_overlay = self.config.show_overlay
1988
+
1989
+ image_path = self.image_map.resolve(image_path)
1990
+
1991
+ # If backtrack is True, use the individual backtrack system
1992
+ if backtrack:
1993
+ return self._execute_with_individual_backtrack(
1994
+ "click_image",
1995
+ self._click_image_internal,
1996
+ image_path,
1997
+ region,
1998
+ confidence,
1999
+ delay,
2000
+ mouse_button,
2001
+ max_attempts,
2002
+ specific,
2003
+ sendtext,
2004
+ show_overlay,
2005
+ wait_until_found,
2006
+ wait_until_disappears,
2007
+ wait_timeout,
2008
+ occurrence,
2009
+ backtrack=backtrack,
2010
+ )
2011
+
2012
+ # Original behavior without backtrack
2013
+ return self._click_image_internal(
2014
+ image_path,
2015
+ region,
2016
+ confidence,
2017
+ delay,
2018
+ mouse_button,
2019
+ max_attempts,
2020
+ specific,
2021
+ sendtext,
2022
+ show_overlay,
2023
+ wait_until_found,
2024
+ wait_until_disappears,
2025
+ wait_timeout,
2026
+ occurrence,
2027
+ )
2028
+
2029
+ @_records_in_session("click")
2030
+ def click_at(self, location, mouse_button="left", delay=0, show_overlay=None):
2031
+ """
2032
+ Click on specific screen coordinates.
2033
+
2034
+ Args:
2035
+ location (tuple): (x, y, width, height) of the location
2036
+ mouse_button (str): Mouse button ("left", "right", "double", "move_to")
2037
+ delay (float): Delay after the click
2038
+ show_overlay (bool, optional): Whether to show the red overlay before the click
2039
+ If None, uses the global configuration
2040
+
2041
+ Returns:
2042
+ bool: True if clicked successfully
2043
+
2044
+ Examples:
2045
+ >>> bot.click_at((100, 200, 50, 30), mouse_button='left', delay=1)
2046
+ """
2047
+ try:
2048
+ # Use the global configuration if not specified
2049
+ if show_overlay is None:
2050
+ show_overlay = self.config.show_overlay
2051
+
2052
+ # Build a temporary task to use the executor
2053
+ temp_task = {
2054
+ "mouse_button": mouse_button,
2055
+ "delay": delay,
2056
+ "show_overlay": show_overlay,
2057
+ }
2058
+
2059
+ self.executor._perform_action(temp_task, location)
2060
+ return True
2061
+
2062
+ except Exception as e:
2063
+ logger.error(f"Error clicking on coordinates {location}: {e}")
2064
+ return False
2065
+
2066
+ def find_relative_image(
2067
+ self,
2068
+ anchor_image,
2069
+ target_image,
2070
+ max_distance=200,
2071
+ confidence=0.9,
2072
+ target_region=None,
2073
+ wait_until_found=False,
2074
+ wait_until_disappears=False,
2075
+ wait_timeout=None,
2076
+ ):
2077
+ """
2078
+ Find a target image near an anchor image.
2079
+
2080
+ Args:
2081
+ anchor_image (str): Path to the anchor image (unique on screen)
2082
+ target_image (str): Path to the target image (there may be multiple)
2083
+ max_distance (int): Maximum distance in pixels from the anchor to the target
2084
+ confidence (float): Confidence level for detection (0.0-1.0)
2085
+ target_region (tuple, optional): Specific region to search for the target (x, y, width, height)
2086
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
2087
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
2088
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
2089
+
2090
+ Returns:
2091
+ tuple: Location of the target image closest to the anchor, or None
2092
+
2093
+ Examples:
2094
+ >>> location = bot.find_relative_image('anchor.png', 'target.png', max_distance=150)
2095
+ >>> if location:
2096
+ ... print(f"Target found near the anchor: {location}")
2097
+ """
2098
+ # If there are wait parameters, build a task and use the executor with wait
2099
+ if wait_until_found or wait_until_disappears:
2100
+ # Build a search function for the wait manager
2101
+ def search_relative():
2102
+ try:
2103
+ return self.executor.relative_detector.locate_relative_image(
2104
+ anchor_image,
2105
+ target_image,
2106
+ confidence,
2107
+ max_distance,
2108
+ target_region,
2109
+ )
2110
+ except Exception:
2111
+ return None
2112
+
2113
+ if wait_until_found:
2114
+ return self.executor.wait_manager.wait_until_found(
2115
+ search_relative,
2116
+ timeout=wait_timeout or self.config.get("wait_timeout", 30),
2117
+ description=f"relative image '{target_image}' near '{anchor_image}'",
2118
+ )
2119
+ elif wait_until_disappears:
2120
+ # First find the image
2121
+ result = search_relative()
2122
+ if not result:
2123
+ return None
2124
+
2125
+ # Then wait for it to disappear
2126
+ disappeared = self.executor.wait_manager.wait_until_disappears(
2127
+ search_relative,
2128
+ timeout=wait_timeout or self.config.get("wait_timeout", 30),
2129
+ description=f"relative image '{target_image}' near '{anchor_image}'",
2130
+ )
2131
+ return result if disappeared else None
2132
+
2133
+ # Original behavior without wait
2134
+ try:
2135
+ return self.executor.relative_detector.locate_relative_image(
2136
+ anchor_image, target_image, confidence, max_distance, target_region
2137
+ )
2138
+ except Exception as e:
2139
+ logger.error(f"Error during relative image search: {e}")
2140
+ return None
2141
+
2142
+ @_records_in_session("relative_image")
2143
+ def click_relative_image(
2144
+ self,
2145
+ anchor_image,
2146
+ target_image,
2147
+ max_distance=200,
2148
+ confidence=0.9,
2149
+ target_region=None,
2150
+ delay=0,
2151
+ mouse_button="left",
2152
+ backtrack=False,
2153
+ max_attempts=3,
2154
+ sendtext=None,
2155
+ wait_until_found=False,
2156
+ wait_until_disappears=False,
2157
+ wait_timeout=None,
2158
+ ):
2159
+ """
2160
+ Click on a target image near an anchor image.
2161
+
2162
+ Args:
2163
+ anchor_image (str): Path to the anchor image
2164
+ target_image (str): Path to the target image
2165
+ max_distance (int): Maximum distance in pixels
2166
+ confidence (float): Confidence level (0.0-1.0)
2167
+ target_region (tuple, optional): Region to search for the target (x, y, width, height)
2168
+ delay (float): Delay after the click in seconds
2169
+ mouse_button (str): Mouse click type:
2170
+ • "left" - Simple left click (default)
2171
+ • "right" - Right click
2172
+ • "double" or "double left" - Double left click
2173
+ • "move_to" - Just moves the mouse without clicking
2174
+ backtrack (bool): Whether to use backtracking on failure
2175
+ max_attempts (int): Maximum number of attempts
2176
+ sendtext (str, optional): Text to type after the click
2177
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
2178
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
2179
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
2180
+
2181
+ Returns:
2182
+ bool: True if found and clicked, False otherwise
2183
+
2184
+ Examples:
2185
+ >>> # Simple left click (default)
2186
+ >>> success = bot.click_relative_image('anchor.png', 'target.png',
2187
+ ... max_distance=150, mouse_button="left")
2188
+
2189
+ >>> # Right click
2190
+ >>> success = bot.click_relative_image('anchor.png', 'target.png',
2191
+ ... mouse_button="right", backtrack=True)
2192
+
2193
+ >>> # Double left click
2194
+ >>> success = bot.click_relative_image('anchor.png', 'target.png',
2195
+ ... mouse_button="double", delay=1.0)
2196
+
2197
+ >>> # Just move the mouse to the position (without clicking)
2198
+ >>> success = bot.click_relative_image('anchor.png', 'target.png',
2199
+ ... mouse_button="move_to", delay=0.5)
2200
+
2201
+ >>> # Click and type text afterwards
2202
+ >>> success = bot.click_relative_image('anchor.png', 'input.png',
2203
+ ... sendtext="Hello World", delay=1.0)
2204
+ """
2205
+ if backtrack:
2206
+ return self._execute_with_individual_backtrack(
2207
+ "click_relative_image",
2208
+ self._click_relative_image_internal,
2209
+ anchor_image,
2210
+ target_image,
2211
+ max_distance,
2212
+ confidence,
2213
+ target_region,
2214
+ delay,
2215
+ mouse_button,
2216
+ max_attempts,
2217
+ sendtext,
2218
+ wait_until_found,
2219
+ wait_until_disappears,
2220
+ wait_timeout,
2221
+ backtrack=backtrack,
2222
+ )
2223
+
2224
+ return self._click_relative_image_internal(
2225
+ anchor_image,
2226
+ target_image,
2227
+ max_distance,
2228
+ confidence,
2229
+ target_region,
2230
+ delay,
2231
+ mouse_button,
2232
+ max_attempts,
2233
+ sendtext,
2234
+ wait_until_found,
2235
+ wait_until_disappears,
2236
+ wait_timeout,
2237
+ )
2238
+
2239
+ def _click_relative_image_internal(
2240
+ self,
2241
+ anchor_image,
2242
+ target_image,
2243
+ max_distance,
2244
+ confidence,
2245
+ target_region,
2246
+ delay,
2247
+ mouse_button,
2248
+ max_attempts,
2249
+ sendtext=None,
2250
+ wait_until_found=False,
2251
+ wait_until_disappears=False,
2252
+ wait_timeout=None,
2253
+ ):
2254
+ """Internal implementation of the relative image click."""
2255
+ for attempt in range(max_attempts):
2256
+ # For click_relative_image, wait_until_disappears must wait AFTER the click, not before
2257
+ # So we pass only wait_until_found to find_relative_image
2258
+ location = self.find_relative_image(
2259
+ anchor_image,
2260
+ target_image,
2261
+ max_distance,
2262
+ confidence,
2263
+ target_region,
2264
+ wait_until_found,
2265
+ False,
2266
+ wait_timeout,
2267
+ ) # wait_until_disappears=False
2268
+ if location:
2269
+ temp_task = {
2270
+ "mouse_button": mouse_button,
2271
+ "delay": delay,
2272
+ "sendtext": sendtext,
2273
+ "wait_until_disappears": wait_until_disappears,
2274
+ "wait_timeout": wait_timeout,
2275
+ "image": target_image, # So wait_until_disappears knows which image to wait for
2276
+ "_wait_after_click": wait_until_disappears,
2277
+ "_wait_timeout": wait_timeout,
2278
+ "_image_path": target_image,
2279
+ }
2280
+ self.executor._perform_action(temp_task, location)
2281
+ return True
2282
+
2283
+ if attempt < max_attempts - 1:
2284
+ time.sleep(0.5)
2285
+
2286
+ return False
2287
+
2288
+ @_records_in_session("click")
2289
+ def click_coordinates(self, x, y, delay=0, mouse_button="left", backtrack=False):
2290
+ """
2291
+ Click on specific screen coordinates.
2292
+
2293
+ Args:
2294
+ x (int): X coordinate
2295
+ y (int): Y coordinate
2296
+ delay (float): Delay after the click
2297
+ mouse_button (str): Mouse button ("left", "right", "double", "move_to")
2298
+ backtrack (bool): Whether to use backtracking
2299
+
2300
+ Returns:
2301
+ bool: True if clicked successfully
2302
+
2303
+ Examples:
2304
+ >>> success = bot.click_coordinates(100, 200, delay=1, backtrack=True)
2305
+ """
2306
+ if backtrack:
2307
+ return self._execute_with_individual_backtrack(
2308
+ "click_coordinates",
2309
+ self._click_coordinates_internal,
2310
+ x,
2311
+ y,
2312
+ delay,
2313
+ mouse_button,
2314
+ backtrack=backtrack,
2315
+ )
2316
+
2317
+ return self._click_coordinates_internal(x, y, delay, mouse_button)
2318
+
2319
+ def _click_coordinates_internal(self, x, y, delay, mouse_button):
2320
+ """Internal implementation of the coordinate click."""
2321
+ try:
2322
+ location = (x, y, 1, 1) # Create a small region
2323
+ temp_task = {"mouse_button": mouse_button, "delay": delay}
2324
+ self.executor._perform_action(temp_task, location)
2325
+ return True
2326
+ except Exception as e:
2327
+ logger.error(f"Error clicking on coordinates ({x}, {y}): {e}")
2328
+ return False
2329
+
2330
+ @_records_in_session("type_text")
2331
+ def type_text(self, text, interval=0.05, delay=0, backtrack=False):
2332
+ """
2333
+ Type text with an interval between characters.
2334
+
2335
+ Args:
2336
+ text (str): Text to type
2337
+ interval (float): Interval between characters in seconds
2338
+ delay (float): Delay after typing
2339
+ backtrack (bool): Whether to use backtracking
2340
+
2341
+ Returns:
2342
+ bool: True if typed successfully
2343
+
2344
+ Examples:
2345
+ >>> success = bot.type_text('Hello World!', interval=0.05, backtrack=True)
2346
+ >>> success = bot.type_text('{ctrl}a{del}New text', backtrack=True)
2347
+ """
2348
+ if backtrack:
2349
+ return self._execute_with_individual_backtrack(
2350
+ "type_text",
2351
+ self._type_text_internal,
2352
+ text,
2353
+ interval,
2354
+ delay,
2355
+ backtrack=backtrack,
2356
+ )
2357
+
2358
+ return self._type_text_internal(text, interval, delay)
2359
+
2360
+ def _type_text_internal(self, text, interval, delay):
2361
+ """Internal implementation of text typing."""
2362
+ try:
2363
+ # Check whether there are special commands
2364
+ if any(
2365
+ cmd in text.lower() for cmd in ["{ctrl}", "{del}", "{tab}", "{enter}"]
2366
+ ):
2367
+ self.executor.keyboard_commander.process_sendtext_command(text)
2368
+ else:
2369
+ self.executor.keyboard_commander.type_text(text, interval)
2370
+
2371
+ if delay > 0:
2372
+ time.sleep(delay)
2373
+ return True
2374
+ except Exception as e:
2375
+ logger.error(f"Error typing text '{text}': {e}")
2376
+ return False
2377
+
2378
+ @_records_in_session("keyboard_command")
2379
+ def keyboard_command(self, command, delay=0, backtrack=False):
2380
+ """
2381
+ Run a keyboard command.
2382
+
2383
+ Args:
2384
+ command (str): Name of the command to run
2385
+ delay (float): Delay after the command
2386
+ backtrack (bool): Whether to use backtracking
2387
+
2388
+ Returns:
2389
+ bool: True if the command ran successfully
2390
+
2391
+ Examples:
2392
+ >>> success = bot.keyboard_command('Ctrl+S', delay=1, backtrack=True)
2393
+ >>> success = bot.keyboard_command('F7', backtrack=True) # Oracle Forms
2394
+ """
2395
+ if backtrack:
2396
+ return self._execute_with_individual_backtrack(
2397
+ "keyboard_command",
2398
+ self._keyboard_command_internal,
2399
+ command,
2400
+ delay,
2401
+ backtrack=backtrack,
2402
+ )
2403
+
2404
+ return self._keyboard_command_internal(command, delay)
2405
+
2406
+ def _keyboard_command_internal(self, command, delay):
2407
+ """Internal implementation of the keyboard command."""
2408
+ try:
2409
+ success = self.executor.keyboard_commander.execute_command(command)
2410
+ if delay > 0:
2411
+ time.sleep(delay)
2412
+ return success
2413
+ except Exception as e:
2414
+ logger.error(f"Error running command '{command}': {e}")
2415
+ return False
2416
+
2417
+ def get_available_keyboard_commands(self):
2418
+ """
2419
+ Return the list of available keyboard commands.
2420
+
2421
+ Returns:
2422
+ list: List of all available commands
2423
+
2424
+ Examples:
2425
+ >>> commands = bot.get_available_keyboard_commands()
2426
+ >>> print(f"Available commands: {len(commands)}")
2427
+ >>> for cmd in commands[:10]: # Show the first 10
2428
+ ... print(f" - {cmd}")
2429
+ """
2430
+ return self.executor.keyboard_commander.get_available_commands()
2431
+
2432
+ def _process_sendtext(self, sendtext):
2433
+ """
2434
+ Process special commands in sendtext and type the text.
2435
+
2436
+ Args:
2437
+ sendtext (str): Text with special commands such as {ctrl}a, {del}, etc.
2438
+ """
2439
+ import time
2440
+
2441
+ import pyautogui
2442
+ import pyperclip
2443
+
2444
+ text_to_write = sendtext
2445
+ commands_processed = True
2446
+
2447
+ while commands_processed:
2448
+ commands_processed = False
2449
+ lower_text = text_to_write.lower()
2450
+
2451
+ if lower_text.startswith("{ctrl}a"):
2452
+ pyautogui.hotkey("ctrl", "a")
2453
+ text_to_write = text_to_write[7:] # Remove '{ctrl}a'
2454
+ commands_processed = True
2455
+ time.sleep(0.1)
2456
+
2457
+ elif lower_text.startswith("{del}"):
2458
+ pyautogui.press("delete")
2459
+ text_to_write = text_to_write[5:] # Remove '{del}'
2460
+ commands_processed = True
2461
+ time.sleep(0.1)
2462
+
2463
+ elif lower_text.startswith("{tab}"):
2464
+ pyautogui.press("tab")
2465
+ text_to_write = text_to_write[5:] # Remove '{tab}'
2466
+ commands_processed = True
2467
+ time.sleep(0.1)
2468
+
2469
+ elif lower_text.startswith("{enter}"):
2470
+ pyautogui.press("enter")
2471
+ text_to_write = text_to_write[7:] # Remove '{enter}'
2472
+ commands_processed = True
2473
+ time.sleep(0.1)
2474
+
2475
+ # Type the remaining text using the clipboard for greater reliability
2476
+ if text_to_write:
2477
+ try:
2478
+ pyperclip.copy(text_to_write)
2479
+ pyautogui.hotkey("ctrl", "v")
2480
+ except Exception:
2481
+ # Fallback to direct typing
2482
+ pyautogui.write(text_to_write)
2483
+ time.sleep(0.1)
2484
+
2485
+ def execute_with_backtrack_between_tasks(self, tasks_list):
2486
+ """
2487
+ Execute multiple individual tasks with real backtracking between them.
2488
+
2489
+ When a task fails, it goes back to the previous one, runs it,
2490
+ and then returns to try the failed one again.
2491
+
2492
+ Args:
2493
+ tasks_list (list): List of dictionaries with individual task settings
2494
+ Each item must have: {'type': 'text'|'image', 'params': {...}}
2495
+
2496
+ Returns:
2497
+ list: List of boolean results
2498
+ """
2499
+ if not tasks_list:
2500
+ return []
2501
+
2502
+ results = []
2503
+ i = 0
2504
+ backtrack_stack = []
2505
+ task_failures = {}
2506
+
2507
+ logger.info(
2508
+ f"🚀 Starting execution with backtracking across {len(tasks_list)} individual tasks"
2509
+ )
2510
+
2511
+ while i < len(tasks_list):
2512
+ task_config = tasks_list[i]
2513
+ task_type = task_config.get("type")
2514
+ params = task_config.get("params", {})
2515
+ backtrack_enabled = params.get("backtrack", True)
2516
+
2517
+ logger.info(
2518
+ f"📋 Running task {i + 1}/{len(tasks_list)} - Type: {task_type}"
2519
+ )
2520
+
2521
+ success = False
2522
+
2523
+ try:
2524
+ if task_type == "text":
2525
+ # Remove backtrack from params to avoid recursion
2526
+ text_params = params.copy()
2527
+ text_params.pop("backtrack", None)
2528
+ success = self.click_text(**text_params)
2529
+
2530
+ elif task_type == "image":
2531
+ # Remove backtrack from params to avoid recursion
2532
+ image_params = params.copy()
2533
+ image_params.pop("backtrack", None)
2534
+ success = self.click_image(**image_params)
2535
+
2536
+ else:
2537
+ logger.error(f"Unknown task type: {task_type}")
2538
+ success = False
2539
+
2540
+ except Exception as e:
2541
+ logger.error(f"Error running task {i + 1}: {e}")
2542
+ success = False
2543
+
2544
+ # Adjust the results list
2545
+ while len(results) <= i:
2546
+ results.append(False)
2547
+ results[i] = success
2548
+
2549
+ if success:
2550
+ logger.info(f"✓ Task {i + 1}/{len(tasks_list)} executed successfully!")
2551
+
2552
+ # Check whether there are pending backtracks on the stack
2553
+ if backtrack_stack:
2554
+ # Pop the item from the stack and go back to the original task that failed
2555
+ original_failed_task = backtrack_stack.pop()
2556
+ logger.info(
2557
+ f"🔄 Returning to task {original_failed_task + 1} that originally failed after backtrack"
2558
+ )
2559
+ i = original_failed_task
2560
+ # Reset the failure attempts for the original task
2561
+ if i in task_failures:
2562
+ task_failures[i] = 0
2563
+ else:
2564
+ # Normal behavior: advance to the next task
2565
+ i += 1
2566
+
2567
+ else:
2568
+ # Task failed, check backtracking
2569
+ if backtrack_enabled and i > 0:
2570
+ # Manage backtracking
2571
+ task_failures.setdefault(i, 0)
2572
+ task_failures[i] += 1
2573
+
2574
+ if task_failures[i] <= 2: # Limit backtrack attempts
2575
+ # Add the current task to the backtrack stack (to return later)
2576
+ if i not in backtrack_stack: # Avoid duplicates
2577
+ backtrack_stack.append(i)
2578
+ logger.info(
2579
+ f"📌 Task {i + 1} added to the backtrack stack for later re-execution"
2580
+ )
2581
+
2582
+ prev_task_type = tasks_list[i - 1].get("type", "unknown")
2583
+ logger.info(
2584
+ f"✗ Task {i + 1}/{len(tasks_list)} ({task_type}) failed. "
2585
+ f"BACKTRACKING to task {i}/{len(tasks_list)} ({prev_task_type})"
2586
+ )
2587
+ i -= 1 # Go back to the previous task (THIS IS THE KEY LINE!)
2588
+ else:
2589
+ logger.info(
2590
+ f"✗ Task {i + 1}/{len(tasks_list)} failed "
2591
+ f"after multiple backtracking attempts. Advancing."
2592
+ )
2593
+ # Remove from the stack if it is there
2594
+ if i in backtrack_stack:
2595
+ backtrack_stack.remove(i)
2596
+ i += 1
2597
+ else:
2598
+ # No backtrack or first task
2599
+ if backtrack_enabled:
2600
+ logger.info(
2601
+ f"✗ Task {i + 1}/{len(tasks_list)} failed "
2602
+ f"but it is the first task. Advancing."
2603
+ )
2604
+ else:
2605
+ logger.info(
2606
+ f"✗ Task {i + 1}/{len(tasks_list)} failed "
2607
+ f"and has 'backtrack': False. Advancing."
2608
+ )
2609
+ i += 1
2610
+
2611
+ successful_tasks = sum(1 for r in results if r)
2612
+ logger.info(
2613
+ f"🏁 Execution finished: {successful_tasks}/{len(results)} successful tasks"
2614
+ )
2615
+
2616
+ return results
2617
+
2618
+
2619
+ # Backward-compatibility alias (previous branding: bot-vision-suite / import bot_vision)
2620
+ BotVision = VisusBot
2621
+
2622
+
2623
+ # NEW CONVENIENCE FUNCTIONS WITH INDIVIDUAL BACKTRACK
2624
+
2625
+
2626
+ def start_individual_session(config=None):
2627
+ """
2628
+ Start an individual task session with backtracking.
2629
+
2630
+ Args:
2631
+ config (dict, optional): Custom configuration
2632
+
2633
+ Returns:
2634
+ VisusBot: Instance configured for individual backtracking
2635
+ """
2636
+ bot = VisusBot(config)
2637
+ bot.start_task_session()
2638
+ return bot
2639
+
2640
+
2641
+ def run_individual_tasks_with_backtrack(task_functions, config=None):
2642
+ """
2643
+ Run a list of individual task functions with automatic backtracking.
2644
+
2645
+ Args:
2646
+ task_functions (list): List of tuples (function, args, kwargs)
2647
+ config (dict, optional): Custom configuration
2648
+
2649
+ Returns:
2650
+ tuple: (successes, total, bot_instance)
2651
+
2652
+ Examples:
2653
+ >>> tasks = [
2654
+ ... (lambda bot: bot.click_image('btn.png', backtrack=True), (), {}),
2655
+ ... (lambda bot: bot.click_text('Save', backtrack=True), (), {}),
2656
+ ... ]
2657
+ >>> success, total, bot = run_individual_tasks_with_backtrack(tasks)
2658
+ """
2659
+ bot = VisusBot(config)
2660
+ bot.start_task_session()
2661
+
2662
+ results = []
2663
+ for i, (func, args, kwargs) in enumerate(task_functions):
2664
+ try:
2665
+ result = func(bot, *args, **kwargs)
2666
+ results.append(result)
2667
+ logger.info(f"Task {i + 1}: {'✓ Success' if result else '✗ Failed'}")
2668
+ except Exception as e:
2669
+ logger.error(f"Error in task {i + 1}: {e}")
2670
+ results.append(False)
2671
+
2672
+ successful, total = bot.end_task_session()
2673
+ return successful, total, bot
2674
+
2675
+
2676
+ # Main function compatible with legacy code
2677
+ def execute_tasks(tasks, config=None):
2678
+ """
2679
+ Main function to run a list of tasks.
2680
+
2681
+ Supports both simple lists and lists of lists (multiple sets).
2682
+ This is the function that users migrated from the original code will use.
2683
+
2684
+ Args:
2685
+ tasks (list): List of automation tasks or list of lists
2686
+ config (dict, optional): Custom configuration
2687
+
2688
+ Returns:
2689
+ list: List of TaskResult or list of lists of TaskResult
2690
+
2691
+ Examples:
2692
+ Simple list:
2693
+ >>> tasks = [{'text': 'Login', 'region': (100, 100, 500, 300)}]
2694
+ >>> results = execute_tasks(tasks)
2695
+
2696
+ Multiple lists:
2697
+ >>> task_sets = [
2698
+ ... [{'text': 'Login', 'region': (100, 100, 500, 300)}],
2699
+ ... [{'text': 'Save', 'region': (200, 200, 600, 400)}]
2700
+ ... ]
2701
+ >>> results = execute_tasks(task_sets)
2702
+ """
2703
+ if not isinstance(tasks, list):
2704
+ logger.error(f"Tasks must be a list. Received type: {type(tasks)}")
2705
+ return []
2706
+
2707
+ if not tasks:
2708
+ logger.info("Empty task list.")
2709
+ return []
2710
+
2711
+ # Check whether it is a list of lists
2712
+ if isinstance(tasks[0], list):
2713
+ # Multiple task sets
2714
+ logger.info(
2715
+ f"Detected multiple task sets ({len(tasks)} sets). Running sequentially."
2716
+ )
2717
+
2718
+ all_results = []
2719
+ for i, task_list in enumerate(tasks):
2720
+ if isinstance(task_list, list):
2721
+ logger.info(
2722
+ f"--- Starting task set {i + 1}/{len(tasks)} ({len(task_list)} tasks) ---"
2723
+ )
2724
+ bot = VisusBot(config)
2725
+ results = bot.execute_tasks(task_list)
2726
+ all_results.append(results)
2727
+ logger.info(f"--- Finished task set {i + 1}/{len(tasks)} ---")
2728
+ else:
2729
+ logger.warning(
2730
+ f"Item {i} in the main list is not a list of tasks. Skipping."
2731
+ )
2732
+ all_results.append([])
2733
+
2734
+ return all_results
2735
+ else:
2736
+ # Simple list of tasks
2737
+ bot = VisusBot(config)
2738
+ return bot.execute_tasks(tasks)
2739
+
2740
+
2741
+ # Additional function for full compatibility with the original click_images
2742
+ def click_images(tasks, default_confidence=0.9, default_margin=50):
2743
+ """
2744
+ Function for full compatibility with legacy code.
2745
+
2746
+ This function replicates exactly the behavior of the click_images function
2747
+ from the original bot_vision.py, including support for lists of lists.
2748
+
2749
+ Args:
2750
+ tasks: List of tasks or list of lists of tasks
2751
+ default_confidence (float): Default confidence for image detection
2752
+ default_margin (int): Default margin (kept for compatibility)
2753
+
2754
+ Returns:
2755
+ list: List of results
2756
+ """
2757
+ # Build a configuration with compatible default values
2758
+ config = {
2759
+ "default_confidence": default_confidence,
2760
+ "default_margin": default_margin,
2761
+ "tesseract_path": r"C:\Program Files\Tesseract-OCR\tesseract.exe",
2762
+ "tessdata_path": r"C:\Program Files\Tesseract-OCR\tessdata",
2763
+ }
2764
+
2765
+ return execute_tasks(tasks, config)
2766
+
2767
+
2768
+ # Standalone functions for 100% compatibility with the original code
2769
+ def locate_image_with_retry(
2770
+ image_path: str, region=None, confidence=0.9, max_attempts=3, scales=None
2771
+ ):
2772
+ """
2773
+ Standalone function that replicates exactly the original function from bot_vision.py.
2774
+ Tries to locate an image with different scales and confidence levels.
2775
+ """
2776
+ executor = TaskExecutor()
2777
+ return executor._locate_image_with_retry(
2778
+ image_path, region, confidence, max_attempts, scales
2779
+ )
2780
+
2781
+
2782
+ # Standalone convenience functions - NEW FEATURES
2783
+ def find_relative_image(
2784
+ anchor_image,
2785
+ target_image,
2786
+ max_distance=200,
2787
+ confidence=0.9,
2788
+ target_region=None,
2789
+ wait_until_found=False,
2790
+ wait_until_disappears=False,
2791
+ wait_timeout=None,
2792
+ ):
2793
+ """
2794
+ Standalone function to find a relative image.
2795
+
2796
+ Args:
2797
+ anchor_image (str): Path to the anchor image
2798
+ target_image (str): Path to the target image
2799
+ max_distance (int): Maximum distance in pixels
2800
+ confidence (float): Confidence level
2801
+ target_region (tuple, optional): Region to search for the target
2802
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
2803
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
2804
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
2805
+
2806
+ Returns:
2807
+ tuple: Location of the image or None
2808
+
2809
+ Examples:
2810
+ >>> location = find_relative_image('anchor.png', 'target.png')
2811
+ """
2812
+ bot = VisusBot()
2813
+ return bot.find_relative_image(
2814
+ anchor_image,
2815
+ target_image,
2816
+ max_distance,
2817
+ confidence,
2818
+ target_region,
2819
+ wait_until_found,
2820
+ wait_until_disappears,
2821
+ wait_timeout,
2822
+ )
2823
+
2824
+
2825
+ def click_relative_image(
2826
+ anchor_image,
2827
+ target_image,
2828
+ max_distance=200,
2829
+ confidence=0.9,
2830
+ target_region=None,
2831
+ delay=0,
2832
+ mouse_button="left",
2833
+ backtrack=False,
2834
+ wait_until_found=False,
2835
+ wait_until_disappears=False,
2836
+ wait_timeout=None,
2837
+ ):
2838
+ """
2839
+ Standalone function to click on a relative image.
2840
+
2841
+ Args:
2842
+ anchor_image (str): Path to the anchor image
2843
+ target_image (str): Path to the target image
2844
+ max_distance (int): Maximum distance in pixels
2845
+ confidence (float): Confidence level (0.0-1.0)
2846
+ target_region (tuple, optional): Region to search for the target (x, y, width, height)
2847
+ delay (float): Delay after the click in seconds
2848
+ mouse_button (str): Mouse click type:
2849
+ • "left" - Simple left click (default)
2850
+ • "right" - Right click
2851
+ • "double" or "double left" - Double left click
2852
+ • "move_to" - Just moves the mouse without clicking
2853
+ backtrack (bool): Whether to use backtracking on failure
2854
+
2855
+ Returns:
2856
+ bool: True if clicked successfully, False otherwise
2857
+
2858
+ Examples:
2859
+ >>> # Simple left click
2860
+ >>> success = click_relative_image('anchor.png', 'target.png')
2861
+
2862
+ >>> # Right click
2863
+ >>> success = click_relative_image('anchor.png', 'target.png',
2864
+ ... mouse_button="right", backtrack=True)
2865
+
2866
+ >>> # Double left click
2867
+ >>> success = click_relative_image('anchor.png', 'target.png',
2868
+ ... mouse_button="double", delay=1.0)
2869
+
2870
+ >>> # Just move the mouse to the position (without clicking)
2871
+ >>> success = click_relative_image('anchor.png', 'target.png',
2872
+ ... mouse_button="move_to", delay=0.5)
2873
+
2874
+ Additional args v1.3.0+:
2875
+ wait_until_found (bool): Waits for the image to appear automatically
2876
+ wait_until_disappears (bool): Waits for the image to disappear
2877
+ wait_timeout (int, optional): Specific timeout (overrides the global one)
2878
+ """
2879
+ bot = VisusBot()
2880
+ return bot.click_relative_image(
2881
+ anchor_image,
2882
+ target_image,
2883
+ max_distance,
2884
+ confidence,
2885
+ target_region,
2886
+ delay,
2887
+ mouse_button,
2888
+ backtrack,
2889
+ 3,
2890
+ wait_until_found,
2891
+ wait_until_disappears,
2892
+ wait_timeout,
2893
+ )
2894
+
2895
+
2896
+ def click_coordinates(x, y, delay=0, mouse_button="left", backtrack=False):
2897
+ """
2898
+ Standalone function to click on specific coordinates.
2899
+
2900
+ Args:
2901
+ x (int): X coordinate
2902
+ y (int): Y coordinate
2903
+ delay (float): Delay after the click in seconds
2904
+ mouse_button (str): Mouse click type:
2905
+ • "left" - Simple left click (default)
2906
+ • "right" - Right click
2907
+ • "double" or "double left" - Double left click
2908
+ • "move_to" - Just moves the mouse without clicking
2909
+ backtrack (bool): Whether to use backtracking on failure
2910
+
2911
+ Returns:
2912
+ bool: True if clicked successfully, False otherwise
2913
+
2914
+ Examples:
2915
+ >>> # Simple click
2916
+ >>> success = click_coordinates(100, 200, delay=1)
2917
+
2918
+ >>> # Right click
2919
+ >>> success = click_coordinates(100, 200, mouse_button="right", backtrack=True)
2920
+
2921
+ >>> # Just move the mouse to the position (without clicking)
2922
+ >>> success = click_coordinates(100, 200, mouse_button="move_to", delay=0.5)
2923
+ """
2924
+ bot = VisusBot()
2925
+ return bot.click_coordinates(x, y, delay, mouse_button, backtrack)
2926
+
2927
+
2928
+ def type_text_standalone(text, interval=0.05, delay=0, backtrack=False):
2929
+ """
2930
+ Standalone function to type text.
2931
+
2932
+ Args:
2933
+ text (str): Text to type
2934
+ interval (float): Interval between characters
2935
+ delay (float): Delay after typing
2936
+ backtrack (bool): Whether to use backtracking
2937
+
2938
+ Returns:
2939
+ bool: True if typed successfully
2940
+
2941
+ Examples:
2942
+ >>> success = type_text_standalone('Hello World!', backtrack=True)
2943
+ >>> success = type_text_standalone('{ctrl}a{del}New text', backtrack=True)
2944
+ """
2945
+ bot = VisusBot()
2946
+ return bot.type_text(text, interval, delay, backtrack)
2947
+
2948
+
2949
+ def keyboard_command_standalone(command, delay=0, backtrack=False):
2950
+ """
2951
+ Standalone function to run a keyboard command.
2952
+
2953
+ Args:
2954
+ command (str): Command to run
2955
+ delay (float): Delay after the command
2956
+ backtrack (bool): Whether to use backtracking
2957
+
2958
+ Returns:
2959
+ bool: True if it ran successfully
2960
+
2961
+ Examples:
2962
+ >>> success = keyboard_command_standalone('Ctrl+S', delay=1, backtrack=True)
2963
+ >>> success = keyboard_command_standalone('F7', backtrack=True)
2964
+ """
2965
+ bot = VisusBot()
2966
+ return bot.keyboard_command(command, delay, backtrack)
2967
+
2968
+
2969
+ def get_available_keyboard_commands():
2970
+ """
2971
+ Standalone function to get the available keyboard commands.
2972
+
2973
+ Returns:
2974
+ list: List of available commands
2975
+
2976
+ Examples:
2977
+ >>> commands = get_available_keyboard_commands()
2978
+ >>> print(f"Total commands: {len(commands)}")
2979
+ """
2980
+ bot = VisusBot()
2981
+ return bot.get_available_keyboard_commands()
2982
+
2983
+
2984
+ def extract_text_from_region_standalone(
2985
+ region=None,
2986
+ filter_type="both",
2987
+ confidence_threshold=50.0,
2988
+ return_full_data=False,
2989
+ max_attempts=3,
2990
+ backtrack=False,
2991
+ config=None,
2992
+ ):
2993
+ """
2994
+ Standalone function to extract text from a specific region of the screen.
2995
+
2996
+ Args:
2997
+ region (tuple, optional): (x, y, width, height) of the region to extract text from.
2998
+ If None, extracts from the whole screen
2999
+ filter_type (str): Filter type ("numbers", "letters", "both")
3000
+ confidence_threshold (float): Minimum confidence threshold (0-100)
3001
+ return_full_data (bool): If True, returns full data with coordinates
3002
+ max_attempts (int): Maximum number of attempts if backtrack=True
3003
+ backtrack (bool): Whether to try multiple times with adjustments
3004
+ config (dict or BotVisionConfig, optional): Custom configuration
3005
+
3006
+ Returns:
3007
+ If return_full_data=False:
3008
+ list: List of strings with all the text found
3009
+ If return_full_data=True:
3010
+ list: List of dictionaries with full data
3011
+
3012
+ Examples:
3013
+ Simple extraction:
3014
+ >>> texts = extract_text_from_region_standalone(region=(100, 100, 400, 300))
3015
+ >>> print("Texts found:", texts)
3016
+
3017
+ Extraction with full data:
3018
+ >>> data = extract_text_from_region_standalone(
3019
+ ... region=(100, 100, 400, 300),
3020
+ ... return_full_data=True,
3021
+ ... confidence_threshold=70.0,
3022
+ ... backtrack=True
3023
+ ... )
3024
+ >>> for item in data:
3025
+ ... print(f"'{item['text']}' - Confidence: {item['confidence']:.1f}%")
3026
+
3027
+ Numbers only:
3028
+ >>> numbers = extract_text_from_region_standalone(
3029
+ ... region=(50, 50, 200, 100),
3030
+ ... filter_type="numbers"
3031
+ ... )
3032
+ """
3033
+ bot = VisusBot(config=config)
3034
+ return bot.extract_text_from_region(
3035
+ region=region,
3036
+ filter_type=filter_type,
3037
+ confidence_threshold=confidence_threshold,
3038
+ return_full_data=return_full_data,
3039
+ max_attempts=max_attempts,
3040
+ backtrack=backtrack,
3041
+ )
3042
+
3043
+
3044
+ # Function for full compatibility - task execution with multiple-list support
3045
+ def run_automation(tasks, default_confidence=0.9, default_margin=50):
3046
+ """
3047
+ Function that replicates the exact behavior of the if __name__ == '__main__' block of the original bot_vision.py.
3048
+ Supports both a simple list and a list of lists (multiple task sets).
3049
+
3050
+ Args:
3051
+ tasks: List of tasks or list of lists of tasks (as in the original)
3052
+ default_confidence (float): Default confidence
3053
+ default_margin (int): Default margin
3054
+ """
3055
+ import logging
3056
+
3057
+ # Check whether tasks is a list and not empty
3058
+ if isinstance(tasks, list) and tasks:
3059
+ # Check whether the first element is also a list (indicating a list of lists)
3060
+ if isinstance(tasks[0], list):
3061
+ logging.info(
3062
+ f"Detected multiple task lists ({len(tasks)} lists). Executing sequentially."
3063
+ )
3064
+ # Iterate through each task list
3065
+ for i, task_list in enumerate(tasks):
3066
+ if isinstance(task_list, list):
3067
+ logging.info(
3068
+ f"--- Starting task list {i + 1}/{len(tasks)} ({len(task_list)} tasks) ---"
3069
+ )
3070
+ click_images(
3071
+ task_list, default_confidence, default_margin
3072
+ ) # Pass the individual list to the function
3073
+ logging.info(f"--- Finished task list {i + 1}/{len(tasks)} ---")
3074
+ else:
3075
+ logging.warning(
3076
+ f"Item {i} in the main list is not a list of tasks. Skipping."
3077
+ )
3078
+ else:
3079
+ # Assume it is a single flat list of tasks
3080
+ logging.info("Detected a single task list. Executing.")
3081
+ click_images(tasks, default_confidence, default_margin)
3082
+ elif isinstance(tasks, list) and not tasks:
3083
+ logging.info("The imported 'tasks' list is empty. Nothing to execute.")
3084
+ else:
3085
+ logging.error(
3086
+ f"The imported 'tasks' is not a list. Type: {type(tasks)}. Cannot execute."
3087
+ )
3088
+
3089
+
3090
+ # Initial library setup
3091
+ def _setup_library():
3092
+ """Initial library setup."""
3093
+ try:
3094
+ # Configure logging if it is not already configured
3095
+ if not logging.getLogger().handlers:
3096
+ logging.basicConfig(
3097
+ level=logging.INFO, format="[%(levelname)s] %(name)s: %(message)s"
3098
+ )
3099
+
3100
+ # Try to configure the library with the default configuration
3101
+ get_default_config()
3102
+ logger.debug(f"visus.desktop v{__version__} initialized")
3103
+
3104
+ except Exception as e:
3105
+ logger.warning(f"Warning during library initialization: {e}")
3106
+
3107
+
3108
+ # Run the initial setup
3109
+ _setup_library()
3110
+
3111
+
3112
+ # Standalone convenience functions (create a temporary VisusBot instance)
3113
+ def find_text(
3114
+ text,
3115
+ region=None,
3116
+ filter_type="both",
3117
+ confidence_threshold=75.0,
3118
+ occurrence=1,
3119
+ max_attempts=3,
3120
+ config=None,
3121
+ wait_until_found=False,
3122
+ wait_until_disappears=False,
3123
+ wait_timeout=None,
3124
+ ):
3125
+ """
3126
+ Search for text on screen using a temporary VisusBot instance.
3127
+
3128
+ Args:
3129
+ text (str): Text to find
3130
+ region (tuple, optional): Search region (x, y, width, height)
3131
+ filter_type (str): Filter type ("letters", "numbers", "both")
3132
+ confidence_threshold (float): OCR confidence threshold
3133
+ occurrence (int): Which occurrence to look for (1=first)
3134
+ max_attempts (int): Maximum number of attempts
3135
+ config (dict, optional): Custom configuration
3136
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
3137
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
3138
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
3139
+
3140
+ Returns:
3141
+ tuple: Found coordinates or None
3142
+ """
3143
+ bot = VisusBot(config)
3144
+ return bot.find_text(
3145
+ text,
3146
+ region,
3147
+ filter_type,
3148
+ confidence_threshold,
3149
+ occurrence,
3150
+ max_attempts,
3151
+ wait_until_found=wait_until_found,
3152
+ wait_until_disappears=wait_until_disappears,
3153
+ wait_timeout=wait_timeout,
3154
+ )
3155
+
3156
+
3157
+ def click_text(
3158
+ text,
3159
+ region=None,
3160
+ filter_type="both",
3161
+ delay=0,
3162
+ mouse_button="left",
3163
+ occurrence=1,
3164
+ max_attempts=3,
3165
+ sendtext=None,
3166
+ confidence_threshold=None,
3167
+ show_overlay=True,
3168
+ config=None,
3169
+ wait_until_found=False,
3170
+ wait_until_disappears=False,
3171
+ wait_timeout=None,
3172
+ ):
3173
+ """
3174
+ Click on text using a temporary VisusBot instance.
3175
+
3176
+ Args:
3177
+ text (str): Text to click on
3178
+ region (tuple, optional): Search region (x, y, width, height)
3179
+ filter_type (str): Filter type ("letters", "numbers", "both")
3180
+ delay (float): Delay after the click in seconds
3181
+ mouse_button (str): Mouse click type:
3182
+ • "left" - Simple left click (default)
3183
+ • "right" - Right click
3184
+ • "double" or "double left" - Double left click
3185
+ • "move_to" - Just moves the mouse without clicking
3186
+ occurrence (int): Which occurrence to click (1=first)
3187
+ max_attempts (int): Maximum number of attempts
3188
+ sendtext (str, optional): Text to type after the click
3189
+ confidence_threshold (float, optional): Custom confidence threshold
3190
+ show_overlay (bool): Whether to show the red overlay before the click
3191
+ config (dict, optional): Custom configuration
3192
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the text to appear automatically
3193
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the text to disappear
3194
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
3195
+
3196
+ Returns:
3197
+ bool: True if found and clicked, False otherwise
3198
+ """
3199
+ bot = VisusBot(config)
3200
+ return bot.click_text(
3201
+ text,
3202
+ region,
3203
+ filter_type,
3204
+ delay,
3205
+ mouse_button,
3206
+ occurrence,
3207
+ False,
3208
+ max_attempts,
3209
+ sendtext,
3210
+ confidence_threshold,
3211
+ show_overlay,
3212
+ wait_until_found,
3213
+ wait_until_disappears,
3214
+ wait_timeout,
3215
+ )
3216
+
3217
+
3218
+ def find_image(
3219
+ image_path,
3220
+ region=None,
3221
+ confidence=0.9,
3222
+ max_attempts=3,
3223
+ specific=True,
3224
+ scales=None,
3225
+ config=None,
3226
+ wait_until_found=False,
3227
+ wait_until_disappears=False,
3228
+ wait_timeout=None,
3229
+ occurrence=1,
3230
+ ):
3231
+ """
3232
+ Search for an image on screen using a temporary VisusBot instance.
3233
+
3234
+ Args:
3235
+ image_path (str): Path to the image
3236
+ region (tuple, optional): Search region
3237
+ confidence (float): Confidence level
3238
+ max_attempts (int): Maximum number of attempts
3239
+ specific (bool): If True, searches in the region; if False, searches the whole screen + variations
3240
+ scales (list, optional): List of scales to try
3241
+ config (dict, optional): Custom configuration
3242
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
3243
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
3244
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
3245
+ occurrence (int): Which match to return when the template appears more
3246
+ than once (1=first, reading order top→bottom/left→right)
3247
+
3248
+ Returns:
3249
+ tuple: Coordinates of the image or None
3250
+ """
3251
+ bot = VisusBot(config)
3252
+ return bot.find_image(
3253
+ image_path,
3254
+ region,
3255
+ confidence,
3256
+ max_attempts,
3257
+ False,
3258
+ specific,
3259
+ wait_until_found,
3260
+ wait_until_disappears,
3261
+ wait_timeout,
3262
+ scales,
3263
+ occurrence=occurrence,
3264
+ )
3265
+
3266
+
3267
+ def click_image(
3268
+ image_path,
3269
+ region=None,
3270
+ confidence=0.9,
3271
+ delay=0,
3272
+ mouse_button="left",
3273
+ max_attempts=3,
3274
+ specific=True,
3275
+ sendtext=None,
3276
+ show_overlay=True,
3277
+ config=None,
3278
+ wait_until_found=False,
3279
+ wait_until_disappears=False,
3280
+ wait_timeout=None,
3281
+ occurrence=1,
3282
+ ):
3283
+ """
3284
+ Click on an image using a temporary VisusBot instance.
3285
+
3286
+ Args:
3287
+ image_path (str): Path to the image
3288
+ region (tuple, optional): Search region (x, y, width, height)
3289
+ confidence (float): Confidence level (0.0-1.0)
3290
+ delay (float): Delay after the click in seconds
3291
+ mouse_button (str): Mouse click type:
3292
+ • "left" - Simple left click (default)
3293
+ • "right" - Right click
3294
+ • "double" or "double left" - Double left click
3295
+ • "move_to" - Just moves the mouse without clicking
3296
+ max_attempts (int): Maximum number of attempts
3297
+ specific (bool): If True, searches in the region; if False, searches the whole screen + variations
3298
+ sendtext (str, optional): Text to type after the click
3299
+ show_overlay (bool): Whether to show the red overlay before the click
3300
+ config (dict, optional): Custom configuration
3301
+ wait_until_found (bool): 🆕 v1.3.0+ - Waits for the image to appear automatically
3302
+ wait_until_disappears (bool): 🆕 v1.3.0+ - Waits for the image to disappear
3303
+ wait_timeout (int, optional): 🆕 v1.3.0+ - Specific timeout (overrides the global one)
3304
+ occurrence (int): Which match to click when the template appears more
3305
+ than once (1=first, reading order top→bottom/left→right)
3306
+
3307
+ Returns:
3308
+ bool: True if found and clicked, False otherwise
3309
+ """
3310
+ bot = VisusBot(config)
3311
+ return bot.click_image(
3312
+ image_path,
3313
+ region,
3314
+ confidence,
3315
+ delay,
3316
+ mouse_button,
3317
+ max_attempts,
3318
+ False,
3319
+ specific,
3320
+ sendtext,
3321
+ show_overlay,
3322
+ wait_until_found,
3323
+ wait_until_disappears,
3324
+ wait_timeout,
3325
+ occurrence=occurrence,
3326
+ )
3327
+
3328
+
3329
+ def click_at(location, mouse_button="left", delay=0, show_overlay=True, config=None):
3330
+ """
3331
+ Click on specific coordinates using a temporary VisusBot instance.
3332
+
3333
+ Args:
3334
+ location (tuple): (x, y, width, height) of the location
3335
+ mouse_button (str): Mouse button ("left", "right", "double", "move_to")
3336
+ delay (float): Delay after the click
3337
+ show_overlay (bool): Whether to show the red overlay before the click
3338
+ config (dict, optional): Custom configuration
3339
+
3340
+ Returns:
3341
+ bool: True if clicked successfully
3342
+ """
3343
+ bot = VisusBot(config)
3344
+ return bot.click_at(location, mouse_button, delay, show_overlay)
3345
+
3346
+
3347
+ def type_text(text, config=None):
3348
+ """
3349
+ Type text using a temporary VisusBot instance.
3350
+
3351
+ Args:
3352
+ text (str): Text to type (supports special commands)
3353
+ config (dict, optional): Custom configuration
3354
+
3355
+ Returns:
3356
+ bool: True if typed successfully
3357
+ """
3358
+ bot = VisusBot(config)
3359
+ return bot.type_text(text)
3360
+
3361
+
3362
+ # PUBLIC EXPORTS
3363
+ __all__ = [
3364
+ # Main class
3365
+ "VisusBot",
3366
+ "BotVision", # backward-compatibility alias
3367
+ # Task execution functions
3368
+ "execute_tasks",
3369
+ "click_images", # Full compatibility with legacy code
3370
+ "run_automation", # Compatibility with if __name__ == '__main__'
3371
+ # Basic convenience functions
3372
+ "find_text",
3373
+ "click_text",
3374
+ "find_image",
3375
+ "click_image",
3376
+ "click_at",
3377
+ "type_text",
3378
+ # NEW FEATURES - Convenience functions for relative images
3379
+ "find_relative_image",
3380
+ "click_relative_image",
3381
+ # NEW FEATURES - Convenience functions for coordinates
3382
+ "click_coordinates",
3383
+ # NEW FEATURES - Convenience functions for the keyboard
3384
+ "type_text_standalone",
3385
+ "keyboard_command_standalone",
3386
+ "get_available_keyboard_commands",
3387
+ # NEW FEATURE - Text extraction
3388
+ "extract_text_from_region_standalone",
3389
+ # Individual backtrack functions
3390
+ "start_individual_session",
3391
+ "run_individual_tasks_with_backtrack",
3392
+ # Core classes (for advanced use)
3393
+ "TaskExecutor",
3394
+ "TaskResult",
3395
+ "OCREngine",
3396
+ "OCRResult",
3397
+ "ImageProcessor",
3398
+ "VisualOverlay",
3399
+ "RelativeImageDetector", # NEW
3400
+ "KeyboardCommander", # NEW
3401
+ # Utilities
3402
+ "BotVisionConfig",
3403
+ "get_default_config",
3404
+ "clean_text",
3405
+ "limpar_texto",
3406
+ "matches_filter",
3407
+ "retry",
3408
+ "Session",
3409
+ # Standalone processing functions (compatibility)
3410
+ "preprocess_image_for_ocr",
3411
+ "find_text_with_multiple_preprocessing",
3412
+ "show_overlay",
3413
+ "locate_image_with_retry",
3414
+ # Files / clipboard / screen / process / window / image map / mouse (BotCity parity)
3415
+ "wait_for_file",
3416
+ "wait_for_file_stable",
3417
+ "copy_to_clipboard",
3418
+ "paste",
3419
+ "get_clipboard",
3420
+ "capture_screen",
3421
+ "save_screenshot",
3422
+ "take_screenshot",
3423
+ "get_display_size",
3424
+ "get_monitors",
3425
+ "Monitor",
3426
+ "wait_for_stable_screen",
3427
+ "capture_element",
3428
+ "get_window",
3429
+ "smart_find",
3430
+ "smart_click",
3431
+ "launch",
3432
+ "find_process",
3433
+ "is_process_running",
3434
+ "kill_process",
3435
+ "focus_window",
3436
+ "maximize_window",
3437
+ "minimize_window",
3438
+ "close_window",
3439
+ "ensure_window",
3440
+ "list_windows",
3441
+ "scroll",
3442
+ "drag",
3443
+ "triple_click",
3444
+ "move_random",
3445
+ "mouse_down",
3446
+ "mouse_up",
3447
+ "ImageMap",
3448
+ "find_all",
3449
+ "find_multiple",
3450
+ "Backend",
3451
+ "NativeError",
3452
+ # Exceptions
3453
+ "BotVisionError",
3454
+ "ImageNotFoundError",
3455
+ "TextNotFoundError",
3456
+ "TaskExecutionError",
3457
+ "ImageProcessingError",
3458
+ "OCRProcessingError",
3459
+ "TesseractNotFoundError",
3460
+ "InvalidRegionError",
3461
+ "ConfigurationError",
3462
+ "NotSupportedOnThisOS",
3463
+ # Library information
3464
+ "__version__",
3465
+ "__author__",
3466
+ "__email__",
3467
+ "__license__",
3468
+ ]