AutoMonkey 0.2.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/PKG-INFO +186 -12
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/SOURCES.txt +18 -1
- automonkey-0.4.2/AutoMonkey.egg-info/entry_points.txt +3 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/requires.txt +8 -4
- automonkey-0.4.2/HISTORY.rst +25 -0
- AutoMonkey-0.2.1/README.md → automonkey-0.4.2/PKG-INFO +197 -4
- AutoMonkey-0.2.1/PKG-INFO → automonkey-0.4.2/README.md +155 -23
- automonkey-0.4.2/automonkey/__init__.py +37 -0
- automonkey-0.4.2/automonkey/ai/__init__.py +54 -0
- automonkey-0.4.2/automonkey/ai/__main__.py +76 -0
- automonkey-0.4.2/automonkey/ai/mcp_launch.py +62 -0
- automonkey-0.4.2/automonkey/ai/mcp_server.py +114 -0
- automonkey-0.4.2/automonkey/ai/ocr.py +149 -0
- automonkey-0.4.2/automonkey/ai/screen.py +202 -0
- automonkey-0.4.2/automonkey/ai/script.py +190 -0
- automonkey-0.4.2/automonkey/ai/windows.py +185 -0
- automonkey-0.4.2/automonkey/app_funcs.py +201 -0
- automonkey-0.4.2/automonkey/atspi_funcs.py +1093 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/automonkey.py +60 -7
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/constants.py +17 -0
- automonkey-0.4.2/automonkey/key_funcs.py +279 -0
- automonkey-0.4.2/automonkey/pywinauto_funcs.py +343 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/utils.py +10 -4
- automonkey-0.4.2/pyproject.toml +64 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/requirements.txt +5 -4
- automonkey-0.4.2/setup.cfg +10 -0
- automonkey-0.4.2/setup.py +21 -0
- automonkey-0.4.2/tests/test_ai.py +31 -0
- automonkey-0.4.2/tests/test_atspi_funcs.py +374 -0
- automonkey-0.4.2/tests/test_atspi_integration.py +102 -0
- automonkey-0.4.2/tests/test_key_funcs.py +54 -0
- automonkey-0.4.2/tests/test_mcp_launch.py +47 -0
- automonkey-0.4.2/tests/test_pywinauto_funcs.py +124 -0
- AutoMonkey-0.2.1/AutoMonkey.egg-info/entry_points.txt +0 -2
- AutoMonkey-0.2.1/HISTORY.rst +0 -8
- AutoMonkey-0.2.1/automonkey/__init__.py +0 -7
- AutoMonkey-0.2.1/automonkey/app_funcs.py +0 -181
- AutoMonkey-0.2.1/pyproject.toml +0 -36
- AutoMonkey-0.2.1/setup.cfg +0 -26
- AutoMonkey-0.2.1/setup.py +0 -49
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AUTHORS.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/dependency_links.txt +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/not-zip-safe +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/AutoMonkey.egg-info/top_level.txt +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/CONTRIBUTING.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/LICENSE +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/MANIFEST.in +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/README.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/exceptions.py +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/img_funcs.py +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/mouse_funcs.py +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/automonkey/mouse_tracker.py +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/Makefile +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/authors.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/conf.py +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/contributing.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/history.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/index.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/installation.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/make.bat +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/readme.rst +0 -0
- {AutoMonkey-0.2.1 → automonkey-0.4.2}/docs/usage.rst +0 -0
|
@@ -1,27 +1,49 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: AutoMonkey
|
|
3
|
-
Version: 0.2
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Python Automation using Mouse and Keyboard, for the masses
|
|
5
|
-
Home-page: https://github.com/MihailCosmin/automonkey
|
|
6
|
-
Author: Mihail-Cosmin Munteanu
|
|
7
5
|
Author-email: Mihail-Cosmin Munteanu <munteanumihailcosmin@gmail.com>
|
|
8
|
-
License: GNU General Public License v3
|
|
6
|
+
License: GNU General Public License v3 (GPLv3)
|
|
9
7
|
Project-URL: Homepage, https://github.com/MihailCosmin/AutoMonkey
|
|
10
8
|
Project-URL: Bug Tracker, https://github.com/MihailCosmin/AutoMonkey/issues
|
|
11
9
|
Keywords: automonkey
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
13
|
+
Classifier: Natural Language :: English
|
|
12
14
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier:
|
|
14
|
-
Classifier:
|
|
15
|
-
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
21
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
22
|
+
Requires-Python: >=3.8
|
|
16
23
|
Description-Content-Type: text/markdown
|
|
17
24
|
License-File: LICENSE
|
|
18
25
|
License-File: AUTHORS.rst
|
|
26
|
+
Requires-Dist: pyautogui>=0.9.54
|
|
27
|
+
Requires-Dist: pytesseract>=0.3.9
|
|
28
|
+
Requires-Dist: numpy
|
|
29
|
+
Requires-Dist: keyboard>=0.13.5
|
|
30
|
+
Requires-Dist: clipboard>=0.0.4
|
|
31
|
+
Requires-Dist: opencv-python
|
|
32
|
+
Requires-Dist: pillow>=10.3.0
|
|
33
|
+
Requires-Dist: screeninfo>=0.8
|
|
34
|
+
Requires-Dist: pyperclip==1.8.2
|
|
35
|
+
Requires-Dist: PyScreeze>=0.1.27
|
|
36
|
+
Requires-Dist: pywin32>=303; sys_platform == "win32"
|
|
37
|
+
Requires-Dist: pywin32-ctypes>=0.2.0; sys_platform == "win32"
|
|
38
|
+
Requires-Dist: pywinauto>=0.6.8; sys_platform == "win32"
|
|
39
|
+
Provides-Extra: ai
|
|
40
|
+
Requires-Dist: mcp>=2.0; extra == "ai"
|
|
41
|
+
Dynamic: license-file
|
|
19
42
|
|
|
20
43
|
[](https://www.paypal.com/donate/?hosted_button_id=5Q6TUMXS2QJW4)
|
|
21
44
|
[](https://snyk.io/test/github/MihailCosmin/AutoMonkey)
|
|
22
|
-
[](https://github.com/MihailCosmin/AutoMonkey)
|
|
23
46
|
[](https://github.com/MihailCosmin/AutoMonkey/actions/workflows/python-package.yml)
|
|
24
|
-
[](https://codeclimate.com/github/MihailCosmin/AutoMonkey)
|
|
25
47
|
|
|
26
48
|
# AutoMonkey
|
|
27
49
|
<img alt="AutoMonkey" src="https://github.com/MihailCosmin/AutoMonkey/raw/06181954fa23605583d61843226f5f3997157435/img/monkey.ico" width="100px" height="100px"/>
|
|
@@ -57,6 +79,7 @@ pip install git+https://github.com/MihailCosmin/AutoMonkey
|
|
|
57
79
|
- opencv-python
|
|
58
80
|
- numpy
|
|
59
81
|
- pytesseract
|
|
82
|
+
- <a href="https://github.com/pywinauto/pywinauto">pywinauto</a> (Windows only, for the control actions)
|
|
60
83
|
|
|
61
84
|
# Usage
|
|
62
85
|
Main function to be used is "chain"
|
|
@@ -327,8 +350,159 @@ or
|
|
|
327
350
|
)
|
|
328
351
|
```
|
|
329
352
|
|
|
330
|
-
|
|
331
|
-
|
|
353
|
+
6. Control actions (Windows: <a href="https://github.com/pywinauto/pywinauto">pywinauto</a>; Linux: AT-SPI; macOS: not implemented yet)
|
|
354
|
+
|
|
355
|
+
These find buttons, text boxes and menus through the OS accessibility tree instead of
|
|
356
|
+
coordinates or images, so they keep working when the resolution, theme or window position changes.
|
|
357
|
+
Same function names and arguments on both OSes - a `chain` script using them needs no changes
|
|
358
|
+
to run on either one; only the `backend` argument's value differs (`"uia"`/`"win32"` on Windows,
|
|
359
|
+
`"atspi"` on Linux).
|
|
360
|
+
|
|
361
|
+
Every function takes the window title (regex, matched anywhere in the title, like focus/close)
|
|
362
|
+
and a control:
|
|
363
|
+
* a "best match" name: `"Save"`, `"SaveButton"`, `"Edit"`
|
|
364
|
+
* or a dict of search criteria: `{"title": "Save", "control_type": "Button"}`, `{"auto_id": "num7Button"}`
|
|
365
|
+
|
|
366
|
+
6.1 Windows (pywinauto)
|
|
367
|
+
|
|
368
|
+
Optional for all: `backend="uia"` (default, modern apps) or `"win32"` (older apps), `timeout=10` seconds.
|
|
369
|
+
|
|
370
|
+
In chain the target is a tuple of arguments or a dict of keyword arguments:
|
|
371
|
+
|
|
372
|
+
```python
|
|
373
|
+
from automonkey import chain, list_controls
|
|
374
|
+
|
|
375
|
+
list_controls("Calculator") # prints the controls and the names you can use
|
|
376
|
+
|
|
377
|
+
chain(
|
|
378
|
+
dict(open_app="calc", wait=1),
|
|
379
|
+
dict(wait_control="Calculator"),
|
|
380
|
+
dict(click_control=("Calculator", {"auto_id": "num7Button"})),
|
|
381
|
+
dict(click_control=("Calculator", {"auto_id": "plusButton"})),
|
|
382
|
+
dict(click_control=("Calculator", {"auto_id": "num5Button"})),
|
|
383
|
+
dict(click_control=dict(window="Calculator", control={"auto_id": "equalButton"}, invoke=True)),
|
|
384
|
+
dict(get_control_text=("Calculator", {"auto_id": "CalculatorResults"})), # copied to the clipboard
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
chain(
|
|
388
|
+
dict(open_app="notepad", wait=1),
|
|
389
|
+
dict(type_in_control=dict(window=" - Notepad$", control={"control_type": "Document"}, text="Hello (world)!")),
|
|
390
|
+
dict(select_menu=(" - Notepad$", "Edit->Select all")),
|
|
391
|
+
)
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
Available functions:
|
|
395
|
+
* start_app - start an application with pywinauto
|
|
396
|
+
* click_control, double_click_control, right_click_control - `invoke=True` presses it without moving the mouse
|
|
397
|
+
* type_in_control - types the text as it is; `literal=False` for pywinauto key syntax (`{ENTER}`, `^a`);
|
|
398
|
+
`pause=0.05` if a slow app loses characters
|
|
399
|
+
* set_control_text, get_control_text
|
|
400
|
+
* select_menu - path separated by `->`, e.g. `"File->Save As..."`
|
|
401
|
+
* select_item - item of a combo box / list
|
|
402
|
+
* check_control, uncheck_control
|
|
403
|
+
* wait_control, control_exists
|
|
404
|
+
* list_controls
|
|
405
|
+
* get_window, get_control - return the pywinauto objects for anything else
|
|
406
|
+
|
|
407
|
+
6.2 Linux (AT-SPI)
|
|
408
|
+
|
|
409
|
+
Setup (distro packages, not pip - PyGObject needs system build dependencies pip can't
|
|
410
|
+
guarantee): on Debian/Ubuntu/Raspberry Pi OS,
|
|
411
|
+
|
|
412
|
+
```bash
|
|
413
|
+
sudo apt-get install python3-gi gir1.2-atspi-2.0 at-spi2-core
|
|
414
|
+
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
The `toolkit-accessibility` key only affects apps started *after* it's enabled -
|
|
418
|
+
restart anything already running. Apps must also not have `NO_AT_BRIDGE=1` set in
|
|
419
|
+
their environment, which silently disables AT-SPI registration.
|
|
420
|
+
|
|
421
|
+
Not every app exposes an accessibility tree: GTK, GNOME and LibreOffice apps and
|
|
422
|
+
Firefox work out of the box; Qt apps need `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1` in
|
|
423
|
+
their environment before they start; Chrome/Electron apps need the
|
|
424
|
+
`--force-renderer-accessibility` flag; self-drawn/custom-rendered UIs (games,
|
|
425
|
+
canvas-based apps) can't expose one at all.
|
|
426
|
+
|
|
427
|
+
`control_type` maps to an AT-SPI role, not quite the same names pywinauto uses on
|
|
428
|
+
Windows - confirmed against real GTK widgets, not just the spec (a push button's role
|
|
429
|
+
is `BUTTON`, not `PUSH_BUTTON`, for instance):
|
|
430
|
+
|
|
431
|
+
| control_type | AT-SPI role(s) |
|
|
432
|
+
| --- | --- |
|
|
433
|
+
| `Button` | `BUTTON`, `PUSH_BUTTON`, `TOGGLE_BUTTON` |
|
|
434
|
+
| `CheckBox` | `CHECK_BOX` |
|
|
435
|
+
| `RadioButton` | `RADIO_BUTTON` |
|
|
436
|
+
| `ComboBox` | `COMBO_BOX` |
|
|
437
|
+
| `Edit` | `TEXT`, `ENTRY`, `PASSWORD_TEXT` (also a plain GTK text editor's main area - see note below) |
|
|
438
|
+
| `Document` | `DOCUMENT_TEXT`, `DOCUMENT_FRAME`, `DOCUMENT_WEB`, ... (LibreOffice/WebKit-style content) |
|
|
439
|
+
| `Static` | `LABEL` |
|
|
440
|
+
| `MenuItem` | `MENU_ITEM`, `CHECK_MENU_ITEM`, `RADIO_MENU_ITEM`, `TEAROFF_MENU_ITEM` |
|
|
441
|
+
| `Menu` | `MENU`, `POPUP_MENU` |
|
|
442
|
+
| `MenuBar` | `MENU_BAR` |
|
|
443
|
+
| `List` | `LIST`, `LIST_BOX` |
|
|
444
|
+
| `ListItem` | `LIST_ITEM` |
|
|
445
|
+
| `Tab` / `TabControl` | `PAGE_TAB` / `PAGE_TAB_LIST` |
|
|
446
|
+
| `Window` / `Dialog` | `FRAME` / `DIALOG` |
|
|
447
|
+
|
|
448
|
+
`auto_id` matches the AT-SPI accessible-id or an `"id"` attribute if the app sets one
|
|
449
|
+
(most GTK apps don't), falling back to the accessible name otherwise.
|
|
450
|
+
|
|
451
|
+
Known Wayland limits: AT-SPI tree reading and actions (`invoke=True`, `check_control`,
|
|
452
|
+
`select_menu`, ...) work the same as on X11/Xwayland. What doesn't: mouse clicks
|
|
453
|
+
(`click_control(invoke=False)`) and keyboard injection (`type_in_control`) only reach
|
|
454
|
+
apps running under Xwayland - a native-Wayland client never receives them at all, no
|
|
455
|
+
matter what has focus (confirmed directly: the identical app forced through Xwayland
|
|
456
|
+
via `GDK_BACKEND=x11` works, the native-Wayland instance silently doesn't). Raising/
|
|
457
|
+
activating a window doesn't work either. `click_control(invoke=False)` detects a
|
|
458
|
+
Wayland session and falls back to `invoke=True` with a warning; `type_in_control` has
|
|
459
|
+
no such fallback - use `invoke=True`-based actions where possible, or run the target
|
|
460
|
+
app under Xwayland (`GDK_BACKEND=x11` for GTK apps) if it must be driven by mouse/
|
|
461
|
+
keyboard injection on Wayland.
|
|
462
|
+
|
|
463
|
+
```python
|
|
464
|
+
from automonkey import chain, list_controls
|
|
465
|
+
|
|
466
|
+
list_controls("Calculator") # prints the controls and the names you can use
|
|
467
|
+
|
|
468
|
+
chain(
|
|
469
|
+
# a short wait after each click matters here: Atspi.Action.do_action() returns
|
|
470
|
+
# once the D-Bus call completes, not once the app has processed the resulting
|
|
471
|
+
# GTK signal - clicking straight through with no pause silently drops presses
|
|
472
|
+
# (confirmed directly while building this).
|
|
473
|
+
dict(start_app="gnome-calculator", wait=1),
|
|
474
|
+
dict(wait_control="Calculator"),
|
|
475
|
+
dict(click_control=dict(window="Calculator", control={"title": "7", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
476
|
+
dict(click_control=dict(window="Calculator", control={"title": "+", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
477
|
+
dict(click_control=dict(window="Calculator", control={"title": "5", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
478
|
+
dict(click_control=dict(window="Calculator", control={"title": "=", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
479
|
+
dict(get_control_text=("Calculator", {"control_type": "Edit"})), # copied to the clipboard
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
chain(
|
|
483
|
+
# control_type "Edit" (AT-SPI role "text"), not "Document": gnome-text-editor's
|
|
484
|
+
# main area is a plain GtkSourceView widget, same as most GTK text editors -
|
|
485
|
+
# "Document*" roles are for LibreOffice/WebKit-style content, confirmed against
|
|
486
|
+
# a GtkSourceView-based app while building this.
|
|
487
|
+
dict(start_app="gnome-text-editor", wait=1),
|
|
488
|
+
dict(type_in_control=dict(window="Text Editor", control={"control_type": "Edit"}, text="Hello (world)!")),
|
|
489
|
+
dict(get_control_text=("Text Editor", {"control_type": "Edit"})), # copied to the clipboard
|
|
490
|
+
)
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
`select_menu` isn't shown against gnome-text-editor above: it's a modern GNOME app
|
|
494
|
+
with no traditional File/Edit menu bar at all (confirmed - zero AT-SPI menu-role nodes
|
|
495
|
+
anywhere in its tree), not an AT-SPI limitation. It works the same way as on Windows
|
|
496
|
+
against any app that does have one, e.g. `select_menu("Mousepad", "File->Save As...")`.
|
|
497
|
+
|
|
498
|
+
Same available functions as the Windows list above (`backend="atspi"`, the default on
|
|
499
|
+
Linux, instead of `"uia"`/`"win32"`); `get_window`/`get_control` return `Atspi.Accessible`
|
|
500
|
+
objects instead of pywinauto wrappers for anything not covered here.
|
|
501
|
+
|
|
502
|
+
See `pywinauto_for_linux.md` in the repo for the full implementation notes (role
|
|
503
|
+
mapping, best-match lookup, the AT-SPI-bus-down detection, known gotchas found while
|
|
504
|
+
building this) if you need to go beyond what's documented above.
|
|
505
|
+
|
|
332
506
|
|
|
333
507
|
# Frequently Asked Questions:
|
|
334
508
|
|
|
@@ -18,13 +18,24 @@ AutoMonkey.egg-info/requires.txt
|
|
|
18
18
|
AutoMonkey.egg-info/top_level.txt
|
|
19
19
|
automonkey/__init__.py
|
|
20
20
|
automonkey/app_funcs.py
|
|
21
|
+
automonkey/atspi_funcs.py
|
|
21
22
|
automonkey/automonkey.py
|
|
22
23
|
automonkey/constants.py
|
|
23
24
|
automonkey/exceptions.py
|
|
24
25
|
automonkey/img_funcs.py
|
|
26
|
+
automonkey/key_funcs.py
|
|
25
27
|
automonkey/mouse_funcs.py
|
|
26
28
|
automonkey/mouse_tracker.py
|
|
29
|
+
automonkey/pywinauto_funcs.py
|
|
27
30
|
automonkey/utils.py
|
|
31
|
+
automonkey/ai/__init__.py
|
|
32
|
+
automonkey/ai/__main__.py
|
|
33
|
+
automonkey/ai/mcp_launch.py
|
|
34
|
+
automonkey/ai/mcp_server.py
|
|
35
|
+
automonkey/ai/ocr.py
|
|
36
|
+
automonkey/ai/screen.py
|
|
37
|
+
automonkey/ai/script.py
|
|
38
|
+
automonkey/ai/windows.py
|
|
28
39
|
docs/Makefile
|
|
29
40
|
docs/authors.rst
|
|
30
41
|
docs/conf.py
|
|
@@ -34,4 +45,10 @@ docs/index.rst
|
|
|
34
45
|
docs/installation.rst
|
|
35
46
|
docs/make.bat
|
|
36
47
|
docs/readme.rst
|
|
37
|
-
docs/usage.rst
|
|
48
|
+
docs/usage.rst
|
|
49
|
+
tests/test_ai.py
|
|
50
|
+
tests/test_atspi_funcs.py
|
|
51
|
+
tests/test_atspi_integration.py
|
|
52
|
+
tests/test_key_funcs.py
|
|
53
|
+
tests/test_mcp_launch.py
|
|
54
|
+
tests/test_pywinauto_funcs.py
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
pyautogui>=0.9.
|
|
1
|
+
pyautogui>=0.9.54
|
|
2
2
|
pytesseract>=0.3.9
|
|
3
|
-
numpy
|
|
3
|
+
numpy
|
|
4
4
|
keyboard>=0.13.5
|
|
5
5
|
clipboard>=0.0.4
|
|
6
|
-
opencv-python
|
|
7
|
-
pillow>=
|
|
6
|
+
opencv-python
|
|
7
|
+
pillow>=10.3.0
|
|
8
8
|
screeninfo>=0.8
|
|
9
9
|
pyperclip==1.8.2
|
|
10
10
|
PyScreeze>=0.1.27
|
|
@@ -12,3 +12,7 @@ PyScreeze>=0.1.27
|
|
|
12
12
|
[:sys_platform == "win32"]
|
|
13
13
|
pywin32>=303
|
|
14
14
|
pywin32-ctypes>=0.2.0
|
|
15
|
+
pywinauto>=0.6.8
|
|
16
|
+
|
|
17
|
+
[ai]
|
|
18
|
+
mcp>=2.0
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
=======
|
|
2
|
+
History
|
|
3
|
+
=======
|
|
4
|
+
|
|
5
|
+
0.4.0 (2026-10-13)
|
|
6
|
+
------------------
|
|
7
|
+
|
|
8
|
+
* Add a Linux backend (``automonkey/atspi_funcs.py``) for the control actions
|
|
9
|
+
(``click_control``, ``type_in_control``, ``get_control_text``, ``select_menu``, ...),
|
|
10
|
+
using AT-SPI via ``gi.repository.Atspi`` (``python3-gi``, ``gir1.2-atspi-2.0``,
|
|
11
|
+
``at-spi2-core`` - distro packages, not pip). Same function names and signatures as
|
|
12
|
+
the Windows ``pywinauto_funcs.py`` backend, selected automatically by platform, so a
|
|
13
|
+
``chain`` script using the control actions needs no changes to run on either OS.
|
|
14
|
+
* Known limits on Wayland: mouse clicks and keyboard injection into a control only
|
|
15
|
+
reach apps running under Xwayland, not native Wayland clients; raising/activating a
|
|
16
|
+
window doesn't work either. ``click_control(invoke=False)`` falls back to the AT-SPI
|
|
17
|
+
action and warns when this is detected. See the README's Control actions section.
|
|
18
|
+
* Fix: ``copy()`` (used by ``get_control_text``/clipboard helpers) no longer crashes the
|
|
19
|
+
whole call when one of its two clipboard backends raises - pyperclip's auto-detected
|
|
20
|
+
"gtk" backend does, once ``gi.repository`` has been imported in-process.
|
|
21
|
+
|
|
22
|
+
0.0.2 (2022-11-06)
|
|
23
|
+
------------------
|
|
24
|
+
|
|
25
|
+
* First release on PyPI.
|
|
@@ -1,8 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: AutoMonkey
|
|
3
|
+
Version: 0.4.2
|
|
4
|
+
Summary: Python Automation using Mouse and Keyboard, for the masses
|
|
5
|
+
Author-email: Mihail-Cosmin Munteanu <munteanumihailcosmin@gmail.com>
|
|
6
|
+
License: GNU General Public License v3 (GPLv3)
|
|
7
|
+
Project-URL: Homepage, https://github.com/MihailCosmin/AutoMonkey
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/MihailCosmin/AutoMonkey/issues
|
|
9
|
+
Keywords: automonkey
|
|
10
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
21
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
22
|
+
Requires-Python: >=3.8
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
License-File: AUTHORS.rst
|
|
26
|
+
Requires-Dist: pyautogui>=0.9.54
|
|
27
|
+
Requires-Dist: pytesseract>=0.3.9
|
|
28
|
+
Requires-Dist: numpy
|
|
29
|
+
Requires-Dist: keyboard>=0.13.5
|
|
30
|
+
Requires-Dist: clipboard>=0.0.4
|
|
31
|
+
Requires-Dist: opencv-python
|
|
32
|
+
Requires-Dist: pillow>=10.3.0
|
|
33
|
+
Requires-Dist: screeninfo>=0.8
|
|
34
|
+
Requires-Dist: pyperclip==1.8.2
|
|
35
|
+
Requires-Dist: PyScreeze>=0.1.27
|
|
36
|
+
Requires-Dist: pywin32>=303; sys_platform == "win32"
|
|
37
|
+
Requires-Dist: pywin32-ctypes>=0.2.0; sys_platform == "win32"
|
|
38
|
+
Requires-Dist: pywinauto>=0.6.8; sys_platform == "win32"
|
|
39
|
+
Provides-Extra: ai
|
|
40
|
+
Requires-Dist: mcp>=2.0; extra == "ai"
|
|
41
|
+
Dynamic: license-file
|
|
42
|
+
|
|
1
43
|
[](https://www.paypal.com/donate/?hosted_button_id=5Q6TUMXS2QJW4)
|
|
2
44
|
[](https://snyk.io/test/github/MihailCosmin/AutoMonkey)
|
|
3
|
-
[](https://github.com/MihailCosmin/AutoMonkey)
|
|
4
46
|
[](https://github.com/MihailCosmin/AutoMonkey/actions/workflows/python-package.yml)
|
|
5
|
-
[](https://codeclimate.com/github/MihailCosmin/AutoMonkey)
|
|
6
47
|
|
|
7
48
|
# AutoMonkey
|
|
8
49
|
<img alt="AutoMonkey" src="https://github.com/MihailCosmin/AutoMonkey/raw/06181954fa23605583d61843226f5f3997157435/img/monkey.ico" width="100px" height="100px"/>
|
|
@@ -38,6 +79,7 @@ pip install git+https://github.com/MihailCosmin/AutoMonkey
|
|
|
38
79
|
- opencv-python
|
|
39
80
|
- numpy
|
|
40
81
|
- pytesseract
|
|
82
|
+
- <a href="https://github.com/pywinauto/pywinauto">pywinauto</a> (Windows only, for the control actions)
|
|
41
83
|
|
|
42
84
|
# Usage
|
|
43
85
|
Main function to be used is "chain"
|
|
@@ -308,8 +350,159 @@ or
|
|
|
308
350
|
)
|
|
309
351
|
```
|
|
310
352
|
|
|
311
|
-
|
|
312
|
-
|
|
353
|
+
6. Control actions (Windows: <a href="https://github.com/pywinauto/pywinauto">pywinauto</a>; Linux: AT-SPI; macOS: not implemented yet)
|
|
354
|
+
|
|
355
|
+
These find buttons, text boxes and menus through the OS accessibility tree instead of
|
|
356
|
+
coordinates or images, so they keep working when the resolution, theme or window position changes.
|
|
357
|
+
Same function names and arguments on both OSes - a `chain` script using them needs no changes
|
|
358
|
+
to run on either one; only the `backend` argument's value differs (`"uia"`/`"win32"` on Windows,
|
|
359
|
+
`"atspi"` on Linux).
|
|
360
|
+
|
|
361
|
+
Every function takes the window title (regex, matched anywhere in the title, like focus/close)
|
|
362
|
+
and a control:
|
|
363
|
+
* a "best match" name: `"Save"`, `"SaveButton"`, `"Edit"`
|
|
364
|
+
* or a dict of search criteria: `{"title": "Save", "control_type": "Button"}`, `{"auto_id": "num7Button"}`
|
|
365
|
+
|
|
366
|
+
6.1 Windows (pywinauto)
|
|
367
|
+
|
|
368
|
+
Optional for all: `backend="uia"` (default, modern apps) or `"win32"` (older apps), `timeout=10` seconds.
|
|
369
|
+
|
|
370
|
+
In chain the target is a tuple of arguments or a dict of keyword arguments:
|
|
371
|
+
|
|
372
|
+
```python
|
|
373
|
+
from automonkey import chain, list_controls
|
|
374
|
+
|
|
375
|
+
list_controls("Calculator") # prints the controls and the names you can use
|
|
376
|
+
|
|
377
|
+
chain(
|
|
378
|
+
dict(open_app="calc", wait=1),
|
|
379
|
+
dict(wait_control="Calculator"),
|
|
380
|
+
dict(click_control=("Calculator", {"auto_id": "num7Button"})),
|
|
381
|
+
dict(click_control=("Calculator", {"auto_id": "plusButton"})),
|
|
382
|
+
dict(click_control=("Calculator", {"auto_id": "num5Button"})),
|
|
383
|
+
dict(click_control=dict(window="Calculator", control={"auto_id": "equalButton"}, invoke=True)),
|
|
384
|
+
dict(get_control_text=("Calculator", {"auto_id": "CalculatorResults"})), # copied to the clipboard
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
chain(
|
|
388
|
+
dict(open_app="notepad", wait=1),
|
|
389
|
+
dict(type_in_control=dict(window=" - Notepad$", control={"control_type": "Document"}, text="Hello (world)!")),
|
|
390
|
+
dict(select_menu=(" - Notepad$", "Edit->Select all")),
|
|
391
|
+
)
|
|
392
|
+
```
|
|
393
|
+
|
|
394
|
+
Available functions:
|
|
395
|
+
* start_app - start an application with pywinauto
|
|
396
|
+
* click_control, double_click_control, right_click_control - `invoke=True` presses it without moving the mouse
|
|
397
|
+
* type_in_control - types the text as it is; `literal=False` for pywinauto key syntax (`{ENTER}`, `^a`);
|
|
398
|
+
`pause=0.05` if a slow app loses characters
|
|
399
|
+
* set_control_text, get_control_text
|
|
400
|
+
* select_menu - path separated by `->`, e.g. `"File->Save As..."`
|
|
401
|
+
* select_item - item of a combo box / list
|
|
402
|
+
* check_control, uncheck_control
|
|
403
|
+
* wait_control, control_exists
|
|
404
|
+
* list_controls
|
|
405
|
+
* get_window, get_control - return the pywinauto objects for anything else
|
|
406
|
+
|
|
407
|
+
6.2 Linux (AT-SPI)
|
|
408
|
+
|
|
409
|
+
Setup (distro packages, not pip - PyGObject needs system build dependencies pip can't
|
|
410
|
+
guarantee): on Debian/Ubuntu/Raspberry Pi OS,
|
|
411
|
+
|
|
412
|
+
```bash
|
|
413
|
+
sudo apt-get install python3-gi gir1.2-atspi-2.0 at-spi2-core
|
|
414
|
+
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
The `toolkit-accessibility` key only affects apps started *after* it's enabled -
|
|
418
|
+
restart anything already running. Apps must also not have `NO_AT_BRIDGE=1` set in
|
|
419
|
+
their environment, which silently disables AT-SPI registration.
|
|
420
|
+
|
|
421
|
+
Not every app exposes an accessibility tree: GTK, GNOME and LibreOffice apps and
|
|
422
|
+
Firefox work out of the box; Qt apps need `QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1` in
|
|
423
|
+
their environment before they start; Chrome/Electron apps need the
|
|
424
|
+
`--force-renderer-accessibility` flag; self-drawn/custom-rendered UIs (games,
|
|
425
|
+
canvas-based apps) can't expose one at all.
|
|
426
|
+
|
|
427
|
+
`control_type` maps to an AT-SPI role, not quite the same names pywinauto uses on
|
|
428
|
+
Windows - confirmed against real GTK widgets, not just the spec (a push button's role
|
|
429
|
+
is `BUTTON`, not `PUSH_BUTTON`, for instance):
|
|
430
|
+
|
|
431
|
+
| control_type | AT-SPI role(s) |
|
|
432
|
+
| --- | --- |
|
|
433
|
+
| `Button` | `BUTTON`, `PUSH_BUTTON`, `TOGGLE_BUTTON` |
|
|
434
|
+
| `CheckBox` | `CHECK_BOX` |
|
|
435
|
+
| `RadioButton` | `RADIO_BUTTON` |
|
|
436
|
+
| `ComboBox` | `COMBO_BOX` |
|
|
437
|
+
| `Edit` | `TEXT`, `ENTRY`, `PASSWORD_TEXT` (also a plain GTK text editor's main area - see note below) |
|
|
438
|
+
| `Document` | `DOCUMENT_TEXT`, `DOCUMENT_FRAME`, `DOCUMENT_WEB`, ... (LibreOffice/WebKit-style content) |
|
|
439
|
+
| `Static` | `LABEL` |
|
|
440
|
+
| `MenuItem` | `MENU_ITEM`, `CHECK_MENU_ITEM`, `RADIO_MENU_ITEM`, `TEAROFF_MENU_ITEM` |
|
|
441
|
+
| `Menu` | `MENU`, `POPUP_MENU` |
|
|
442
|
+
| `MenuBar` | `MENU_BAR` |
|
|
443
|
+
| `List` | `LIST`, `LIST_BOX` |
|
|
444
|
+
| `ListItem` | `LIST_ITEM` |
|
|
445
|
+
| `Tab` / `TabControl` | `PAGE_TAB` / `PAGE_TAB_LIST` |
|
|
446
|
+
| `Window` / `Dialog` | `FRAME` / `DIALOG` |
|
|
447
|
+
|
|
448
|
+
`auto_id` matches the AT-SPI accessible-id or an `"id"` attribute if the app sets one
|
|
449
|
+
(most GTK apps don't), falling back to the accessible name otherwise.
|
|
450
|
+
|
|
451
|
+
Known Wayland limits: AT-SPI tree reading and actions (`invoke=True`, `check_control`,
|
|
452
|
+
`select_menu`, ...) work the same as on X11/Xwayland. What doesn't: mouse clicks
|
|
453
|
+
(`click_control(invoke=False)`) and keyboard injection (`type_in_control`) only reach
|
|
454
|
+
apps running under Xwayland - a native-Wayland client never receives them at all, no
|
|
455
|
+
matter what has focus (confirmed directly: the identical app forced through Xwayland
|
|
456
|
+
via `GDK_BACKEND=x11` works, the native-Wayland instance silently doesn't). Raising/
|
|
457
|
+
activating a window doesn't work either. `click_control(invoke=False)` detects a
|
|
458
|
+
Wayland session and falls back to `invoke=True` with a warning; `type_in_control` has
|
|
459
|
+
no such fallback - use `invoke=True`-based actions where possible, or run the target
|
|
460
|
+
app under Xwayland (`GDK_BACKEND=x11` for GTK apps) if it must be driven by mouse/
|
|
461
|
+
keyboard injection on Wayland.
|
|
462
|
+
|
|
463
|
+
```python
|
|
464
|
+
from automonkey import chain, list_controls
|
|
465
|
+
|
|
466
|
+
list_controls("Calculator") # prints the controls and the names you can use
|
|
467
|
+
|
|
468
|
+
chain(
|
|
469
|
+
# a short wait after each click matters here: Atspi.Action.do_action() returns
|
|
470
|
+
# once the D-Bus call completes, not once the app has processed the resulting
|
|
471
|
+
# GTK signal - clicking straight through with no pause silently drops presses
|
|
472
|
+
# (confirmed directly while building this).
|
|
473
|
+
dict(start_app="gnome-calculator", wait=1),
|
|
474
|
+
dict(wait_control="Calculator"),
|
|
475
|
+
dict(click_control=dict(window="Calculator", control={"title": "7", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
476
|
+
dict(click_control=dict(window="Calculator", control={"title": "+", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
477
|
+
dict(click_control=dict(window="Calculator", control={"title": "5", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
478
|
+
dict(click_control=dict(window="Calculator", control={"title": "=", "control_type": "Button"}, invoke=True), wait=0.3),
|
|
479
|
+
dict(get_control_text=("Calculator", {"control_type": "Edit"})), # copied to the clipboard
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
chain(
|
|
483
|
+
# control_type "Edit" (AT-SPI role "text"), not "Document": gnome-text-editor's
|
|
484
|
+
# main area is a plain GtkSourceView widget, same as most GTK text editors -
|
|
485
|
+
# "Document*" roles are for LibreOffice/WebKit-style content, confirmed against
|
|
486
|
+
# a GtkSourceView-based app while building this.
|
|
487
|
+
dict(start_app="gnome-text-editor", wait=1),
|
|
488
|
+
dict(type_in_control=dict(window="Text Editor", control={"control_type": "Edit"}, text="Hello (world)!")),
|
|
489
|
+
dict(get_control_text=("Text Editor", {"control_type": "Edit"})), # copied to the clipboard
|
|
490
|
+
)
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
`select_menu` isn't shown against gnome-text-editor above: it's a modern GNOME app
|
|
494
|
+
with no traditional File/Edit menu bar at all (confirmed - zero AT-SPI menu-role nodes
|
|
495
|
+
anywhere in its tree), not an AT-SPI limitation. It works the same way as on Windows
|
|
496
|
+
against any app that does have one, e.g. `select_menu("Mousepad", "File->Save As...")`.
|
|
497
|
+
|
|
498
|
+
Same available functions as the Windows list above (`backend="atspi"`, the default on
|
|
499
|
+
Linux, instead of `"uia"`/`"win32"`); `get_window`/`get_control` return `Atspi.Accessible`
|
|
500
|
+
objects instead of pywinauto wrappers for anything not covered here.
|
|
501
|
+
|
|
502
|
+
See `pywinauto_for_linux.md` in the repo for the full implementation notes (role
|
|
503
|
+
mapping, best-match lookup, the AT-SPI-bus-down detection, known gotchas found while
|
|
504
|
+
building this) if you need to go beyond what's documented above.
|
|
505
|
+
|
|
313
506
|
|
|
314
507
|
# Frequently Asked Questions:
|
|
315
508
|
|