AutoMonkey 0.1.9__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/PKG-INFO +185 -10
  2. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/SOURCES.txt +19 -1
  3. automonkey-0.4.1/AutoMonkey.egg-info/entry_points.txt +3 -0
  4. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/requires.txt +8 -4
  5. automonkey-0.4.1/HISTORY.rst +25 -0
  6. AutoMonkey-0.1.9/README.md → automonkey-0.4.1/PKG-INFO +196 -2
  7. AutoMonkey-0.1.9/PKG-INFO → automonkey-0.4.1/README.md +154 -21
  8. automonkey-0.4.1/automonkey/__init__.py +37 -0
  9. automonkey-0.4.1/automonkey/ai/__init__.py +54 -0
  10. automonkey-0.4.1/automonkey/ai/__main__.py +76 -0
  11. automonkey-0.4.1/automonkey/ai/mcp_launch.py +62 -0
  12. automonkey-0.4.1/automonkey/ai/mcp_server.py +114 -0
  13. automonkey-0.4.1/automonkey/ai/ocr.py +149 -0
  14. automonkey-0.4.1/automonkey/ai/screen.py +202 -0
  15. automonkey-0.4.1/automonkey/ai/script.py +190 -0
  16. automonkey-0.4.1/automonkey/ai/windows.py +185 -0
  17. automonkey-0.4.1/automonkey/app_funcs.py +201 -0
  18. automonkey-0.4.1/automonkey/atspi_funcs.py +1093 -0
  19. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/automonkey.py +65 -7
  20. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/constants.py +21 -0
  21. automonkey-0.4.1/automonkey/key_funcs.py +279 -0
  22. automonkey-0.4.1/automonkey/mouse_funcs.py +37 -0
  23. automonkey-0.4.1/automonkey/pywinauto_funcs.py +343 -0
  24. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/utils.py +10 -4
  25. automonkey-0.4.1/pyproject.toml +64 -0
  26. {AutoMonkey-0.1.9 → automonkey-0.4.1}/requirements.txt +5 -4
  27. automonkey-0.4.1/setup.cfg +10 -0
  28. automonkey-0.4.1/setup.py +21 -0
  29. automonkey-0.4.1/tests/test_ai.py +31 -0
  30. automonkey-0.4.1/tests/test_atspi_funcs.py +374 -0
  31. automonkey-0.4.1/tests/test_atspi_integration.py +102 -0
  32. automonkey-0.4.1/tests/test_key_funcs.py +54 -0
  33. automonkey-0.4.1/tests/test_mcp_launch.py +47 -0
  34. automonkey-0.4.1/tests/test_pywinauto_funcs.py +124 -0
  35. AutoMonkey-0.1.9/AutoMonkey.egg-info/entry_points.txt +0 -2
  36. AutoMonkey-0.1.9/HISTORY.rst +0 -8
  37. AutoMonkey-0.1.9/automonkey/__init__.py +0 -7
  38. AutoMonkey-0.1.9/automonkey/app_funcs.py +0 -181
  39. AutoMonkey-0.1.9/pyproject.toml +0 -36
  40. AutoMonkey-0.1.9/setup.cfg +0 -26
  41. AutoMonkey-0.1.9/setup.py +0 -49
  42. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AUTHORS.rst +0 -0
  43. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/dependency_links.txt +0 -0
  44. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/not-zip-safe +0 -0
  45. {AutoMonkey-0.1.9 → automonkey-0.4.1}/AutoMonkey.egg-info/top_level.txt +0 -0
  46. {AutoMonkey-0.1.9 → automonkey-0.4.1}/CONTRIBUTING.rst +0 -0
  47. {AutoMonkey-0.1.9 → automonkey-0.4.1}/LICENSE +0 -0
  48. {AutoMonkey-0.1.9 → automonkey-0.4.1}/MANIFEST.in +0 -0
  49. {AutoMonkey-0.1.9 → automonkey-0.4.1}/README.rst +0 -0
  50. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/exceptions.py +0 -0
  51. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/img_funcs.py +0 -0
  52. {AutoMonkey-0.1.9 → automonkey-0.4.1}/automonkey/mouse_tracker.py +0 -0
  53. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/Makefile +0 -0
  54. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/authors.rst +0 -0
  55. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/conf.py +0 -0
  56. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/contributing.rst +0 -0
  57. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/history.rst +0 -0
  58. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/index.rst +0 -0
  59. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/installation.rst +0 -0
  60. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/make.bat +0 -0
  61. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/readme.rst +0 -0
  62. {AutoMonkey-0.1.9 → automonkey-0.4.1}/docs/usage.rst +0 -0
@@ -1,21 +1,44 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: AutoMonkey
3
- Version: 0.1.9
3
+ Version: 0.4.1
4
4
  Summary: Python Automation using Mouse and Keyboard, for the masses
5
- Home-page: https://github.com/MihailCosmin/automonkey
6
- Author: Mihail-Cosmin Munteanu
7
5
  Author-email: Mihail-Cosmin Munteanu <munteanumihailcosmin@gmail.com>
8
- License: GNU General Public License v3
6
+ License: GNU General Public License v3 (GPLv3)
9
7
  Project-URL: Homepage, https://github.com/MihailCosmin/AutoMonkey
10
8
  Project-URL: Bug Tracker, https://github.com/MihailCosmin/AutoMonkey/issues
11
9
  Keywords: automonkey
10
+ Classifier: Development Status :: 2 - Pre-Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
13
+ Classifier: Natural Language :: English
12
14
  Classifier: Programming Language :: Python :: 3
13
- Classifier: License :: OSI Approved :: MIT License
14
- Classifier: Operating System :: OS Independent
15
- Requires-Python: >=3.6
15
+ Classifier: Programming Language :: Python :: 3.8
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Operating System :: Microsoft :: Windows
21
+ Classifier: Operating System :: POSIX :: Linux
22
+ Requires-Python: >=3.8
16
23
  Description-Content-Type: text/markdown
17
24
  License-File: LICENSE
18
25
  License-File: AUTHORS.rst
26
+ Requires-Dist: pyautogui>=0.9.54
27
+ Requires-Dist: pytesseract>=0.3.9
28
+ Requires-Dist: numpy
29
+ Requires-Dist: keyboard>=0.13.5
30
+ Requires-Dist: clipboard>=0.0.4
31
+ Requires-Dist: opencv-python
32
+ Requires-Dist: pillow>=9.5.0
33
+ Requires-Dist: screeninfo>=0.8
34
+ Requires-Dist: pyperclip==1.8.2
35
+ Requires-Dist: PyScreeze>=0.1.27
36
+ Requires-Dist: pywin32>=303; sys_platform == "win32"
37
+ Requires-Dist: pywin32-ctypes>=0.2.0; sys_platform == "win32"
38
+ Requires-Dist: pywinauto>=0.6.8; sys_platform == "win32"
39
+ Provides-Extra: ai
40
+ Requires-Dist: mcp>=2.0; extra == "ai"
41
+ Dynamic: license-file
19
42
 
20
43
  [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=5Q6TUMXS2QJW4)
21
44
  [![](https://snyk.io/test/github/MihailCosmin/AutoMonkey/badge.svg)](https://snyk.io/test/github/MihailCosmin/AutoMonkey)
@@ -57,6 +80,7 @@ pip install git+https://github.com/MihailCosmin/AutoMonkey
57
80
  - opencv-python
58
81
  - numpy
59
82
  - pytesseract
83
+ - <a href="https://github.com/pywinauto/pywinauto">pywinauto</a> (Windows only, for the control actions)
60
84
 
61
85
  # Usage
62
86
  Main function to be used is "chain"
@@ -327,8 +351,159 @@ or
327
351
  )
328
352
  ```
329
353
 
330
- # Roadmap
331
- 1. Check if possible to add <a href="https://github.com/pywinauto/pywinauto">pyautowin</a> functionality
354
+ 6. Control actions (Windows: <a href="https://github.com/pywinauto/pywinauto">pywinauto</a>; Linux: AT-SPI; macOS: not implemented yet)
355
+
356
+ These find buttons, text boxes and menus through the OS accessibility tree instead of
357
+ coordinates or images, so they keep working when the resolution, theme or window position changes.
358
+ Same function names and arguments on both OSes - a `chain` script using them needs no changes
359
+ to run on either one; only the `backend` argument's value differs (`"uia"`/`"win32"` on Windows,
360
+ `"atspi"` on Linux).
361
+
362
+ Every function takes the window title (regex, matched anywhere in the title, like focus/close)
363
+ and a control:
364
+ * a "best match" name: `"Save"`, `"SaveButton"`, `"Edit"`
365
+ * or a dict of search criteria: `{"title": "Save", "control_type": "Button"}`, `{"auto_id": "num7Button"}`
366
+
367
+ 6.1 Windows (pywinauto)
368
+
369
+ Optional for all: `backend="uia"` (default, modern apps) or `"win32"` (older apps), `timeout=10` seconds.
370
+
371
+ In chain the target is a tuple of arguments or a dict of keyword arguments:
372
+
373
+ ```python
374
+ from automonkey import chain, list_controls
375
+
376
+ list_controls("Calculator") # prints the controls and the names you can use
377
+
378
+ chain(
379
+ dict(open_app="calc", wait=1),
380
+ dict(wait_control="Calculator"),
381
+ dict(click_control=("Calculator", {"auto_id": "num7Button"})),
382
+ dict(click_control=("Calculator", {"auto_id": "plusButton"})),
383
+ dict(click_control=("Calculator", {"auto_id": "num5Button"})),
384
+ dict(click_control=dict(window="Calculator", control={"auto_id": "equalButton"}, invoke=True)),
385
+ dict(get_control_text=("Calculator", {"auto_id": "CalculatorResults"})), # copied to the clipboard
386
+ )
387
+
388
+ chain(
389
+ dict(open_app="notepad", wait=1),
390
+ dict(type_in_control=dict(window=" - Notepad$", control={"control_type": "Document"}, text="Hello (world)!")),
391
+ dict(select_menu=(" - Notepad$", "Edit->Select all")),
392
+ )
393
+ ```
394
+
395
+ Available functions:
396
+ * start_app - start an application with pywinauto
397
+ * click_control, double_click_control, right_click_control - `invoke=True` presses it without moving the mouse
398
+ * type_in_control - types the text as it is; `literal=False` for pywinauto key syntax (`{ENTER}`, `^a`);
399
+ `pause=0.05` if a slow app loses characters
400
+ * set_control_text, get_control_text
401
+ * select_menu - path separated by `->`, e.g. `"File->Save As..."`
402
+ * select_item - item of a combo box / list
403
+ * check_control, uncheck_control
404
+ * wait_control, control_exists
405
+ * list_controls
406
+ * get_window, get_control - return the pywinauto objects for anything else
407
+
408
+ 6.2 Linux (AT-SPI)
409
+
410
+ Setup (distro packages, not pip - PyGObject needs system build dependencies pip can't
411
+ guarantee): on Debian/Ubuntu/Raspberry Pi OS,
412
+
413
+ ```bash
414
+ sudo apt-get install python3-gi gir1.2-atspi-2.0 at-spi2-core
415
+ gsettings set org.gnome.desktop.interface toolkit-accessibility true
416
+ ```
417
+
418
+ The `toolkit-accessibility` key only affects apps started *after* it's enabled -
419
+ restart anything already running. Apps must also not have `NO_AT_BRIDGE=1` set in
420
+ their environment, which silently disables AT-SPI registration.
421
+
422
+ Not every app exposes an accessibility tree: GTK, GNOME and LibreOffice apps and
423
+ Firefox work out of the box; Qt apps need `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1` in
424
+ their environment before they start; Chrome/Electron apps need the
425
+ `--force-renderer-accessibility` flag; self-drawn/custom-rendered UIs (games,
426
+ canvas-based apps) can't expose one at all.
427
+
428
+ `control_type` maps to an AT-SPI role, not quite the same names pywinauto uses on
429
+ Windows - confirmed against real GTK widgets, not just the spec (a push button's role
430
+ is `BUTTON`, not `PUSH_BUTTON`, for instance):
431
+
432
+ | control_type | AT-SPI role(s) |
433
+ | --- | --- |
434
+ | `Button` | `BUTTON`, `PUSH_BUTTON`, `TOGGLE_BUTTON` |
435
+ | `CheckBox` | `CHECK_BOX` |
436
+ | `RadioButton` | `RADIO_BUTTON` |
437
+ | `ComboBox` | `COMBO_BOX` |
438
+ | `Edit` | `TEXT`, `ENTRY`, `PASSWORD_TEXT` (also a plain GTK text editor's main area - see note below) |
439
+ | `Document` | `DOCUMENT_TEXT`, `DOCUMENT_FRAME`, `DOCUMENT_WEB`, ... (LibreOffice/WebKit-style content) |
440
+ | `Static` | `LABEL` |
441
+ | `MenuItem` | `MENU_ITEM`, `CHECK_MENU_ITEM`, `RADIO_MENU_ITEM`, `TEAROFF_MENU_ITEM` |
442
+ | `Menu` | `MENU`, `POPUP_MENU` |
443
+ | `MenuBar` | `MENU_BAR` |
444
+ | `List` | `LIST`, `LIST_BOX` |
445
+ | `ListItem` | `LIST_ITEM` |
446
+ | `Tab` / `TabControl` | `PAGE_TAB` / `PAGE_TAB_LIST` |
447
+ | `Window` / `Dialog` | `FRAME` / `DIALOG` |
448
+
449
+ `auto_id` matches the AT-SPI accessible-id or an `"id"` attribute if the app sets one
450
+ (most GTK apps don't), falling back to the accessible name otherwise.
451
+
452
+ Known Wayland limits: AT-SPI tree reading and actions (`invoke=True`, `check_control`,
453
+ `select_menu`, ...) work the same as on X11/Xwayland. What doesn't: mouse clicks
454
+ (`click_control(invoke=False)`) and keyboard injection (`type_in_control`) only reach
455
+ apps running under Xwayland - a native-Wayland client never receives them at all, no
456
+ matter what has focus (confirmed directly: the identical app forced through Xwayland
457
+ via `GDK_BACKEND=x11` works, the native-Wayland instance silently doesn't). Raising/
458
+ activating a window doesn't work either. `click_control(invoke=False)` detects a
459
+ Wayland session and falls back to `invoke=True` with a warning; `type_in_control` has
460
+ no such fallback - use `invoke=True`-based actions where possible, or run the target
461
+ app under Xwayland (`GDK_BACKEND=x11` for GTK apps) if it must be driven by mouse/
462
+ keyboard injection on Wayland.
463
+
464
+ ```python
465
+ from automonkey import chain, list_controls
466
+
467
+ list_controls("Calculator") # prints the controls and the names you can use
468
+
469
+ chain(
470
+ # a short wait after each click matters here: Atspi.Action.do_action() returns
471
+ # once the D-Bus call completes, not once the app has processed the resulting
472
+ # GTK signal - clicking straight through with no pause silently drops presses
473
+ # (confirmed directly while building this).
474
+ dict(start_app="gnome-calculator", wait=1),
475
+ dict(wait_control="Calculator"),
476
+ dict(click_control=dict(window="Calculator", control={"title": "7", "control_type": "Button"}, invoke=True), wait=0.3),
477
+ dict(click_control=dict(window="Calculator", control={"title": "+", "control_type": "Button"}, invoke=True), wait=0.3),
478
+ dict(click_control=dict(window="Calculator", control={"title": "5", "control_type": "Button"}, invoke=True), wait=0.3),
479
+ dict(click_control=dict(window="Calculator", control={"title": "=", "control_type": "Button"}, invoke=True), wait=0.3),
480
+ dict(get_control_text=("Calculator", {"control_type": "Edit"})), # copied to the clipboard
481
+ )
482
+
483
+ chain(
484
+ # control_type "Edit" (AT-SPI role "text"), not "Document": gnome-text-editor's
485
+ # main area is a plain GtkSourceView widget, same as most GTK text editors -
486
+ # "Document*" roles are for LibreOffice/WebKit-style content, confirmed against
487
+ # a GtkSourceView-based app while building this.
488
+ dict(start_app="gnome-text-editor", wait=1),
489
+ dict(type_in_control=dict(window="Text Editor", control={"control_type": "Edit"}, text="Hello (world)!")),
490
+ dict(get_control_text=("Text Editor", {"control_type": "Edit"})), # copied to the clipboard
491
+ )
492
+ ```
493
+
494
+ `select_menu` isn't shown against gnome-text-editor above: it's a modern GNOME app
495
+ with no traditional File/Edit menu bar at all (confirmed - zero AT-SPI menu-role nodes
496
+ anywhere in its tree), not an AT-SPI limitation. It works the same way as on Windows
497
+ against any app that does have one, e.g. `select_menu("Mousepad", "File->Save As...")`.
498
+
499
+ Same available functions as the Windows list above (`backend="atspi"`, the default on
500
+ Linux, instead of `"uia"`/`"win32"`); `get_window`/`get_control` return `Atspi.Accessible`
501
+ objects instead of pywinauto wrappers for anything not covered here.
502
+
503
+ See `pywinauto_for_linux.md` in the repo for the full implementation notes (role
504
+ mapping, best-match lookup, the AT-SPI-bus-down detection, known gotchas found while
505
+ building this) if you need to go beyond what's documented above.
506
+
332
507
 
333
508
  # Frequently Asked Questions:
334
509
 
@@ -18,12 +18,24 @@ AutoMonkey.egg-info/requires.txt
18
18
  AutoMonkey.egg-info/top_level.txt
19
19
  automonkey/__init__.py
20
20
  automonkey/app_funcs.py
21
+ automonkey/atspi_funcs.py
21
22
  automonkey/automonkey.py
22
23
  automonkey/constants.py
23
24
  automonkey/exceptions.py
24
25
  automonkey/img_funcs.py
26
+ automonkey/key_funcs.py
27
+ automonkey/mouse_funcs.py
25
28
  automonkey/mouse_tracker.py
29
+ automonkey/pywinauto_funcs.py
26
30
  automonkey/utils.py
31
+ automonkey/ai/__init__.py
32
+ automonkey/ai/__main__.py
33
+ automonkey/ai/mcp_launch.py
34
+ automonkey/ai/mcp_server.py
35
+ automonkey/ai/ocr.py
36
+ automonkey/ai/screen.py
37
+ automonkey/ai/script.py
38
+ automonkey/ai/windows.py
27
39
  docs/Makefile
28
40
  docs/authors.rst
29
41
  docs/conf.py
@@ -33,4 +45,10 @@ docs/index.rst
33
45
  docs/installation.rst
34
46
  docs/make.bat
35
47
  docs/readme.rst
36
- docs/usage.rst
48
+ docs/usage.rst
49
+ tests/test_ai.py
50
+ tests/test_atspi_funcs.py
51
+ tests/test_atspi_integration.py
52
+ tests/test_key_funcs.py
53
+ tests/test_mcp_launch.py
54
+ tests/test_pywinauto_funcs.py
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ automonkey-ai = automonkey.ai.__main__:main
3
+ automonkey-mcp = automonkey.ai.mcp_server:main
@@ -1,10 +1,10 @@
1
- pyautogui>=0.9.53
1
+ pyautogui>=0.9.54
2
2
  pytesseract>=0.3.9
3
- numpy>=1.21.5
3
+ numpy
4
4
  keyboard>=0.13.5
5
5
  clipboard>=0.0.4
6
- opencv-python==4.5.5.62
7
- pillow>=9.0.1
6
+ opencv-python
7
+ pillow>=9.5.0
8
8
  screeninfo>=0.8
9
9
  pyperclip==1.8.2
10
10
  PyScreeze>=0.1.27
@@ -12,3 +12,7 @@ PyScreeze>=0.1.27
12
12
  [:sys_platform == "win32"]
13
13
  pywin32>=303
14
14
  pywin32-ctypes>=0.2.0
15
+ pywinauto>=0.6.8
16
+
17
+ [ai]
18
+ mcp>=2.0
@@ -0,0 +1,25 @@
1
+ =======
2
+ History
3
+ =======
4
+
5
+ 0.4.0 (2026-10-13)
6
+ ------------------
7
+
8
+ * Add a Linux backend (``automonkey/atspi_funcs.py``) for the control actions
9
+ (``click_control``, ``type_in_control``, ``get_control_text``, ``select_menu``, ...),
10
+ using AT-SPI via ``gi.repository.Atspi`` (``python3-gi``, ``gir1.2-atspi-2.0``,
11
+ ``at-spi2-core`` - distro packages, not pip). Same function names and signatures as
12
+ the Windows ``pywinauto_funcs.py`` backend, selected automatically by platform, so a
13
+ ``chain`` script using the control actions needs no changes to run on either OS.
14
+ * Known limits on Wayland: mouse clicks and keyboard injection into a control only
15
+ reach apps running under Xwayland, not native Wayland clients; raising/activating a
16
+ window doesn't work either. ``click_control(invoke=False)`` falls back to the AT-SPI
17
+ action and warns when this is detected. See the README's Control actions section.
18
+ * Fix: ``copy()`` (used by ``get_control_text``/clipboard helpers) no longer crashes the
19
+ whole call when one of its two clipboard backends raises - pyperclip's auto-detected
20
+ "gtk" backend does, once ``gi.repository`` has been imported in-process.
21
+
22
+ 0.0.2 (2022-11-06)
23
+ ------------------
24
+
25
+ * First release on PyPI.
@@ -1,3 +1,45 @@
1
+ Metadata-Version: 2.4
2
+ Name: AutoMonkey
3
+ Version: 0.4.1
4
+ Summary: Python Automation using Mouse and Keyboard, for the masses
5
+ Author-email: Mihail-Cosmin Munteanu <munteanumihailcosmin@gmail.com>
6
+ License: GNU General Public License v3 (GPLv3)
7
+ Project-URL: Homepage, https://github.com/MihailCosmin/AutoMonkey
8
+ Project-URL: Bug Tracker, https://github.com/MihailCosmin/AutoMonkey/issues
9
+ Keywords: automonkey
10
+ Classifier: Development Status :: 2 - Pre-Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
13
+ Classifier: Natural Language :: English
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.8
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Operating System :: Microsoft :: Windows
21
+ Classifier: Operating System :: POSIX :: Linux
22
+ Requires-Python: >=3.8
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ License-File: AUTHORS.rst
26
+ Requires-Dist: pyautogui>=0.9.54
27
+ Requires-Dist: pytesseract>=0.3.9
28
+ Requires-Dist: numpy
29
+ Requires-Dist: keyboard>=0.13.5
30
+ Requires-Dist: clipboard>=0.0.4
31
+ Requires-Dist: opencv-python
32
+ Requires-Dist: pillow>=9.5.0
33
+ Requires-Dist: screeninfo>=0.8
34
+ Requires-Dist: pyperclip==1.8.2
35
+ Requires-Dist: PyScreeze>=0.1.27
36
+ Requires-Dist: pywin32>=303; sys_platform == "win32"
37
+ Requires-Dist: pywin32-ctypes>=0.2.0; sys_platform == "win32"
38
+ Requires-Dist: pywinauto>=0.6.8; sys_platform == "win32"
39
+ Provides-Extra: ai
40
+ Requires-Dist: mcp>=2.0; extra == "ai"
41
+ Dynamic: license-file
42
+
1
43
  [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=5Q6TUMXS2QJW4)
2
44
  [![](https://snyk.io/test/github/MihailCosmin/AutoMonkey/badge.svg)](https://snyk.io/test/github/MihailCosmin/AutoMonkey)
3
45
  [![HitCount](https://hits.dwyl.com/MihailCosmin/AutoMonkey.svg?style=flat-square&show=unique)](http://hits.dwyl.com/MihailCosmin/AutoMonkey)
@@ -38,6 +80,7 @@ pip install git+https://github.com/MihailCosmin/AutoMonkey
38
80
  - opencv-python
39
81
  - numpy
40
82
  - pytesseract
83
+ - <a href="https://github.com/pywinauto/pywinauto">pywinauto</a> (Windows only, for the control actions)
41
84
 
42
85
  # Usage
43
86
  Main function to be used is "chain"
@@ -308,8 +351,159 @@ or
308
351
  )
309
352
  ```
310
353
 
311
- # Roadmap
312
- 1. Check if possible to add <a href="https://github.com/pywinauto/pywinauto">pyautowin</a> functionality
354
+ 6. Control actions (Windows: <a href="https://github.com/pywinauto/pywinauto">pywinauto</a>; Linux: AT-SPI; macOS: not implemented yet)
355
+
356
+ These find buttons, text boxes and menus through the OS accessibility tree instead of
357
+ coordinates or images, so they keep working when the resolution, theme or window position changes.
358
+ Same function names and arguments on both OSes - a `chain` script using them needs no changes
359
+ to run on either one; only the `backend` argument's value differs (`"uia"`/`"win32"` on Windows,
360
+ `"atspi"` on Linux).
361
+
362
+ Every function takes the window title (regex, matched anywhere in the title, like focus/close)
363
+ and a control:
364
+ * a "best match" name: `"Save"`, `"SaveButton"`, `"Edit"`
365
+ * or a dict of search criteria: `{"title": "Save", "control_type": "Button"}`, `{"auto_id": "num7Button"}`
366
+
367
+ 6.1 Windows (pywinauto)
368
+
369
+ Optional for all: `backend="uia"` (default, modern apps) or `"win32"` (older apps), `timeout=10` seconds.
370
+
371
+ In chain the target is a tuple of arguments or a dict of keyword arguments:
372
+
373
+ ```python
374
+ from automonkey import chain, list_controls
375
+
376
+ list_controls("Calculator") # prints the controls and the names you can use
377
+
378
+ chain(
379
+ dict(open_app="calc", wait=1),
380
+ dict(wait_control="Calculator"),
381
+ dict(click_control=("Calculator", {"auto_id": "num7Button"})),
382
+ dict(click_control=("Calculator", {"auto_id": "plusButton"})),
383
+ dict(click_control=("Calculator", {"auto_id": "num5Button"})),
384
+ dict(click_control=dict(window="Calculator", control={"auto_id": "equalButton"}, invoke=True)),
385
+ dict(get_control_text=("Calculator", {"auto_id": "CalculatorResults"})), # copied to the clipboard
386
+ )
387
+
388
+ chain(
389
+ dict(open_app="notepad", wait=1),
390
+ dict(type_in_control=dict(window=" - Notepad$", control={"control_type": "Document"}, text="Hello (world)!")),
391
+ dict(select_menu=(" - Notepad$", "Edit->Select all")),
392
+ )
393
+ ```
394
+
395
+ Available functions:
396
+ * start_app - start an application with pywinauto
397
+ * click_control, double_click_control, right_click_control - `invoke=True` presses it without moving the mouse
398
+ * type_in_control - types the text as it is; `literal=False` for pywinauto key syntax (`{ENTER}`, `^a`);
399
+ `pause=0.05` if a slow app loses characters
400
+ * set_control_text, get_control_text
401
+ * select_menu - path separated by `->`, e.g. `"File->Save As..."`
402
+ * select_item - item of a combo box / list
403
+ * check_control, uncheck_control
404
+ * wait_control, control_exists
405
+ * list_controls
406
+ * get_window, get_control - return the pywinauto objects for anything else
407
+
408
+ 6.2 Linux (AT-SPI)
409
+
410
+ Setup (distro packages, not pip - PyGObject needs system build dependencies pip can't
411
+ guarantee): on Debian/Ubuntu/Raspberry Pi OS,
412
+
413
+ ```bash
414
+ sudo apt-get install python3-gi gir1.2-atspi-2.0 at-spi2-core
415
+ gsettings set org.gnome.desktop.interface toolkit-accessibility true
416
+ ```
417
+
418
+ The `toolkit-accessibility` key only affects apps started *after* it's enabled -
419
+ restart anything already running. Apps must also not have `NO_AT_BRIDGE=1` set in
420
+ their environment, which silently disables AT-SPI registration.
421
+
422
+ Not every app exposes an accessibility tree: GTK, GNOME and LibreOffice apps and
423
+ Firefox work out of the box; Qt apps need `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1` in
424
+ their environment before they start; Chrome/Electron apps need the
425
+ `--force-renderer-accessibility` flag; self-drawn/custom-rendered UIs (games,
426
+ canvas-based apps) can't expose one at all.
427
+
428
+ `control_type` maps to an AT-SPI role, not quite the same names pywinauto uses on
429
+ Windows - confirmed against real GTK widgets, not just the spec (a push button's role
430
+ is `BUTTON`, not `PUSH_BUTTON`, for instance):
431
+
432
+ | control_type | AT-SPI role(s) |
433
+ | --- | --- |
434
+ | `Button` | `BUTTON`, `PUSH_BUTTON`, `TOGGLE_BUTTON` |
435
+ | `CheckBox` | `CHECK_BOX` |
436
+ | `RadioButton` | `RADIO_BUTTON` |
437
+ | `ComboBox` | `COMBO_BOX` |
438
+ | `Edit` | `TEXT`, `ENTRY`, `PASSWORD_TEXT` (also a plain GTK text editor's main area - see note below) |
439
+ | `Document` | `DOCUMENT_TEXT`, `DOCUMENT_FRAME`, `DOCUMENT_WEB`, ... (LibreOffice/WebKit-style content) |
440
+ | `Static` | `LABEL` |
441
+ | `MenuItem` | `MENU_ITEM`, `CHECK_MENU_ITEM`, `RADIO_MENU_ITEM`, `TEAROFF_MENU_ITEM` |
442
+ | `Menu` | `MENU`, `POPUP_MENU` |
443
+ | `MenuBar` | `MENU_BAR` |
444
+ | `List` | `LIST`, `LIST_BOX` |
445
+ | `ListItem` | `LIST_ITEM` |
446
+ | `Tab` / `TabControl` | `PAGE_TAB` / `PAGE_TAB_LIST` |
447
+ | `Window` / `Dialog` | `FRAME` / `DIALOG` |
448
+
449
+ `auto_id` matches the AT-SPI accessible-id or an `"id"` attribute if the app sets one
450
+ (most GTK apps don't), falling back to the accessible name otherwise.
451
+
452
+ Known Wayland limits: AT-SPI tree reading and actions (`invoke=True`, `check_control`,
453
+ `select_menu`, ...) work the same as on X11/Xwayland. What doesn't: mouse clicks
454
+ (`click_control(invoke=False)`) and keyboard injection (`type_in_control`) only reach
455
+ apps running under Xwayland - a native-Wayland client never receives them at all, no
456
+ matter what has focus (confirmed directly: the identical app forced through Xwayland
457
+ via `GDK_BACKEND=x11` works, the native-Wayland instance silently doesn't). Raising/
458
+ activating a window doesn't work either. `click_control(invoke=False)` detects a
459
+ Wayland session and falls back to `invoke=True` with a warning; `type_in_control` has
460
+ no such fallback - use `invoke=True`-based actions where possible, or run the target
461
+ app under Xwayland (`GDK_BACKEND=x11` for GTK apps) if it must be driven by mouse/
462
+ keyboard injection on Wayland.
463
+
464
+ ```python
465
+ from automonkey import chain, list_controls
466
+
467
+ list_controls("Calculator") # prints the controls and the names you can use
468
+
469
+ chain(
470
+ # a short wait after each click matters here: Atspi.Action.do_action() returns
471
+ # once the D-Bus call completes, not once the app has processed the resulting
472
+ # GTK signal - clicking straight through with no pause silently drops presses
473
+ # (confirmed directly while building this).
474
+ dict(start_app="gnome-calculator", wait=1),
475
+ dict(wait_control="Calculator"),
476
+ dict(click_control=dict(window="Calculator", control={"title": "7", "control_type": "Button"}, invoke=True), wait=0.3),
477
+ dict(click_control=dict(window="Calculator", control={"title": "+", "control_type": "Button"}, invoke=True), wait=0.3),
478
+ dict(click_control=dict(window="Calculator", control={"title": "5", "control_type": "Button"}, invoke=True), wait=0.3),
479
+ dict(click_control=dict(window="Calculator", control={"title": "=", "control_type": "Button"}, invoke=True), wait=0.3),
480
+ dict(get_control_text=("Calculator", {"control_type": "Edit"})), # copied to the clipboard
481
+ )
482
+
483
+ chain(
484
+ # control_type "Edit" (AT-SPI role "text"), not "Document": gnome-text-editor's
485
+ # main area is a plain GtkSourceView widget, same as most GTK text editors -
486
+ # "Document*" roles are for LibreOffice/WebKit-style content, confirmed against
487
+ # a GtkSourceView-based app while building this.
488
+ dict(start_app="gnome-text-editor", wait=1),
489
+ dict(type_in_control=dict(window="Text Editor", control={"control_type": "Edit"}, text="Hello (world)!")),
490
+ dict(get_control_text=("Text Editor", {"control_type": "Edit"})), # copied to the clipboard
491
+ )
492
+ ```
493
+
494
+ `select_menu` isn't shown against gnome-text-editor above: it's a modern GNOME app
495
+ with no traditional File/Edit menu bar at all (confirmed - zero AT-SPI menu-role nodes
496
+ anywhere in its tree), not an AT-SPI limitation. It works the same way as on Windows
497
+ against any app that does have one, e.g. `select_menu("Mousepad", "File->Save As...")`.
498
+
499
+ Same available functions as the Windows list above (`backend="atspi"`, the default on
500
+ Linux, instead of `"uia"`/`"win32"`); `get_window`/`get_control` return `Atspi.Accessible`
501
+ objects instead of pywinauto wrappers for anything not covered here.
502
+
503
+ See `pywinauto_for_linux.md` in the repo for the full implementation notes (role
504
+ mapping, best-match lookup, the AT-SPI-bus-down detection, known gotchas found while
505
+ building this) if you need to go beyond what's documented above.
506
+
313
507
 
314
508
  # Frequently Asked Questions:
315
509