PyClickerRPA 0.3.1__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PKG-INFO +50 -4
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/PKG-INFO +50 -4
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/SOURCES.txt +9 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/entry_points.txt +1 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/README.md +48 -2
- pyclickerrpa-0.5.0/autoclicker/diagnose.py +179 -0
- pyclickerrpa-0.5.0/autoclicker/gui/lint_dialog.py +53 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/main_window.py +132 -6
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/qt_runner.py +6 -1
- pyclickerrpa-0.5.0/autoclicker/gui/recorder.py +290 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/task_widget.py +13 -4
- pyclickerrpa-0.5.0/autoclicker/lint.py +163 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/models.py +3 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/runner.py +38 -0
- pyclickerrpa-0.5.0/autoclicker/templatize.py +119 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/visual.py +11 -4
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/pyproject.toml +3 -2
- pyclickerrpa-0.5.0/tests/test_diagnose.py +198 -0
- pyclickerrpa-0.5.0/tests/test_diagnose_gui.py +97 -0
- pyclickerrpa-0.5.0/tests/test_lint.py +155 -0
- pyclickerrpa-0.5.0/tests/test_recorder_gui.py +321 -0
- pyclickerrpa-0.5.0/tests/test_templatize.py +133 -0
- pyclickerrpa-0.3.1/autoclicker/gui/recorder.py +0 -163
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/LICENSE +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/dependency_links.txt +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/requires.txt +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/top_level.txt +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/__init__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/adb.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/app.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/capture.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/clicker.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/dpi.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/expr.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/__init__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_connection.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_editor.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_node.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/hotkeys.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/monitor.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/preview_label.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/scheduler_widget.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/snippet_library.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/testkit_widget.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/matcher.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/ocr.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/run.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/__init__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/__main__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/assertions.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/case.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/loader.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/pytest_plugin.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/report.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/runner.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/__init__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/__main__.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/run_manager.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/scheduler.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/serialize.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/server.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/app.js +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/index.html +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/style.css +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/windows.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/setup.cfg +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_adb.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_flow_branch.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_matcher.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_models.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_runner.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_testkit.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_testkit_gui.py +0 -0
- {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_web.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyClickerRPA
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, recorder, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, a smart recorder that records images instead of coordinates, task health checks and failure attribution, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
|
|
5
5
|
Author: Equinox
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/Equinox/PyClickerRPA
|
|
@@ -66,6 +66,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
|
|
|
66
66
|
autoclicker # desktop GUI (flow editor / list editor / test tab)
|
|
67
67
|
autoclicker-run task.json --window "Notepad" # headless run against a window
|
|
68
68
|
autoclicker-run task.json --adb # run against an Android device (adb)
|
|
69
|
+
autoclicker-lint task.json # health-check a task (exit code = error count)
|
|
69
70
|
autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
|
|
70
71
|
autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
|
|
71
72
|
```
|
|
@@ -82,7 +83,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
82
83
|
- **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
|
|
83
84
|
changing — replaces guessed `sleep`s (fast when ready, patient when slow).
|
|
84
85
|
- **~30+ step types**: click image/text/position, click-all, relative/offset
|
|
85
|
-
clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
|
|
86
|
+
clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
|
|
86
87
|
key / type / hotkey, loop / goto / while / do-until, if image/text/color/
|
|
87
88
|
count/var with **true/false branching** (if-else), set-var, read-text→variable,
|
|
88
89
|
activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
|
|
@@ -93,12 +94,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
93
94
|
Drag-and-drop **node flow graph** (condition nodes have **true/false branch
|
|
94
95
|
ports** for visual if-else; a terminal **stop** node ends the flow) and an
|
|
95
96
|
interoperable **step list** with a full
|
|
96
|
-
per-step dialog;
|
|
97
|
+
per-step dialog; snippet library, task
|
|
97
98
|
packages (`.zip` with templates), on-preview coordinate/color picking, template
|
|
98
99
|
crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
|
|
99
100
|
and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
|
|
100
101
|
run visual test suites.
|
|
101
102
|
|
|
103
|
+
### Smart recorder
|
|
104
|
+
|
|
105
|
+
The action **recorder** (global hooks) doesn't just replay coordinates and
|
|
106
|
+
sleeps — with **智能录制 / smart mode** on (default) it records what things *look
|
|
107
|
+
like* instead of where they *were*:
|
|
108
|
+
|
|
109
|
+
- **Clicks become image steps.** A patch is cropped around the click and grown
|
|
110
|
+
(48 → 192 px) until it is both textured enough and **unique on screen**, then
|
|
111
|
+
emitted as a `click_image` step — so the flow still works after the window
|
|
112
|
+
moves or the resolution changes. Identical patches are de-duplicated by
|
|
113
|
+
content hash.
|
|
114
|
+
- **Drags become `drag_image`.** The drag's *start* is anchored to an image and
|
|
115
|
+
the *displacement* is stored, so start and end move together.
|
|
116
|
+
- **Pauses become `wait_stable`.** A long pause is only converted if the
|
|
117
|
+
recorder actually **observed the screen settle** during it; on a permanently
|
|
118
|
+
animating screen it keeps the fixed wait (a `wait_stable` that never settles
|
|
119
|
+
would fail the run).
|
|
120
|
+
|
|
121
|
+
Anything that can't be made robust falls back to the plain coordinate/fixed-wait
|
|
122
|
+
step and says why in the log.
|
|
123
|
+
|
|
124
|
+
## Diagnostics
|
|
125
|
+
|
|
126
|
+
**Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
|
|
127
|
+
statically scans a task and separates *will definitely break* from *works but
|
|
128
|
+
fragile*:
|
|
129
|
+
|
|
130
|
+
- **errors** — missing template / sub-task file, image step with no template,
|
|
131
|
+
loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
|
|
132
|
+
for confirmation before running a task with errors; the CLI exit code is the
|
|
133
|
+
error count.
|
|
134
|
+
- **warnings** — steps that depend on absolute coordinates, long fixed waits,
|
|
135
|
+
a match threshold low enough to hit the wrong thing, templates too small to be
|
|
136
|
+
unique. These are exactly the things the smart recorder now avoids, so the
|
|
137
|
+
check doubles as a "what should I re-record?" list.
|
|
138
|
+
|
|
139
|
+
**Failure attribution.** A run captures a screenshot per step; the first
|
|
140
|
+
successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
|
|
141
|
+
run fails, the frames are compared step by step and the report names the step
|
|
142
|
+
where the screen **first diverged** from the successful run — which is usually
|
|
143
|
+
*earlier* than the step that failed (step 7 can't find its button because a popup
|
|
144
|
+
appeared at step 4). You get the diverging step, the per-step changed-pixel
|
|
145
|
+
ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
|
|
146
|
+
that points at timing/flakiness rather than a UI change.
|
|
147
|
+
|
|
102
148
|
## Visual test framework (`autoclicker.testkit`)
|
|
103
149
|
|
|
104
150
|
Turn automation into tests:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PyClickerRPA
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, recorder, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, a smart recorder that records images instead of coordinates, task health checks and failure attribution, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
|
|
5
5
|
Author: Equinox
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/Equinox/PyClickerRPA
|
|
@@ -66,6 +66,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
|
|
|
66
66
|
autoclicker # desktop GUI (flow editor / list editor / test tab)
|
|
67
67
|
autoclicker-run task.json --window "Notepad" # headless run against a window
|
|
68
68
|
autoclicker-run task.json --adb # run against an Android device (adb)
|
|
69
|
+
autoclicker-lint task.json # health-check a task (exit code = error count)
|
|
69
70
|
autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
|
|
70
71
|
autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
|
|
71
72
|
```
|
|
@@ -82,7 +83,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
82
83
|
- **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
|
|
83
84
|
changing — replaces guessed `sleep`s (fast when ready, patient when slow).
|
|
84
85
|
- **~30+ step types**: click image/text/position, click-all, relative/offset
|
|
85
|
-
clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
|
|
86
|
+
clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
|
|
86
87
|
key / type / hotkey, loop / goto / while / do-until, if image/text/color/
|
|
87
88
|
count/var with **true/false branching** (if-else), set-var, read-text→variable,
|
|
88
89
|
activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
|
|
@@ -93,12 +94,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
93
94
|
Drag-and-drop **node flow graph** (condition nodes have **true/false branch
|
|
94
95
|
ports** for visual if-else; a terminal **stop** node ends the flow) and an
|
|
95
96
|
interoperable **step list** with a full
|
|
96
|
-
per-step dialog;
|
|
97
|
+
per-step dialog; snippet library, task
|
|
97
98
|
packages (`.zip` with templates), on-preview coordinate/color picking, template
|
|
98
99
|
crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
|
|
99
100
|
and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
|
|
100
101
|
run visual test suites.
|
|
101
102
|
|
|
103
|
+
### Smart recorder
|
|
104
|
+
|
|
105
|
+
The action **recorder** (global hooks) doesn't just replay coordinates and
|
|
106
|
+
sleeps — with **智能录制 / smart mode** on (default) it records what things *look
|
|
107
|
+
like* instead of where they *were*:
|
|
108
|
+
|
|
109
|
+
- **Clicks become image steps.** A patch is cropped around the click and grown
|
|
110
|
+
(48 → 192 px) until it is both textured enough and **unique on screen**, then
|
|
111
|
+
emitted as a `click_image` step — so the flow still works after the window
|
|
112
|
+
moves or the resolution changes. Identical patches are de-duplicated by
|
|
113
|
+
content hash.
|
|
114
|
+
- **Drags become `drag_image`.** The drag's *start* is anchored to an image and
|
|
115
|
+
the *displacement* is stored, so start and end move together.
|
|
116
|
+
- **Pauses become `wait_stable`.** A long pause is only converted if the
|
|
117
|
+
recorder actually **observed the screen settle** during it; on a permanently
|
|
118
|
+
animating screen it keeps the fixed wait (a `wait_stable` that never settles
|
|
119
|
+
would fail the run).
|
|
120
|
+
|
|
121
|
+
Anything that can't be made robust falls back to the plain coordinate/fixed-wait
|
|
122
|
+
step and says why in the log.
|
|
123
|
+
|
|
124
|
+
## Diagnostics
|
|
125
|
+
|
|
126
|
+
**Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
|
|
127
|
+
statically scans a task and separates *will definitely break* from *works but
|
|
128
|
+
fragile*:
|
|
129
|
+
|
|
130
|
+
- **errors** — missing template / sub-task file, image step with no template,
|
|
131
|
+
loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
|
|
132
|
+
for confirmation before running a task with errors; the CLI exit code is the
|
|
133
|
+
error count.
|
|
134
|
+
- **warnings** — steps that depend on absolute coordinates, long fixed waits,
|
|
135
|
+
a match threshold low enough to hit the wrong thing, templates too small to be
|
|
136
|
+
unique. These are exactly the things the smart recorder now avoids, so the
|
|
137
|
+
check doubles as a "what should I re-record?" list.
|
|
138
|
+
|
|
139
|
+
**Failure attribution.** A run captures a screenshot per step; the first
|
|
140
|
+
successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
|
|
141
|
+
run fails, the frames are compared step by step and the report names the step
|
|
142
|
+
where the screen **first diverged** from the successful run — which is usually
|
|
143
|
+
*earlier* than the step that failed (step 7 can't find its button because a popup
|
|
144
|
+
appeared at step 4). You get the diverging step, the per-step changed-pixel
|
|
145
|
+
ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
|
|
146
|
+
that points at timing/flakiness rather than a UI change.
|
|
147
|
+
|
|
102
148
|
## Visual test framework (`autoclicker.testkit`)
|
|
103
149
|
|
|
104
150
|
Turn automation into tests:
|
|
@@ -12,19 +12,23 @@ autoclicker/adb.py
|
|
|
12
12
|
autoclicker/app.py
|
|
13
13
|
autoclicker/capture.py
|
|
14
14
|
autoclicker/clicker.py
|
|
15
|
+
autoclicker/diagnose.py
|
|
15
16
|
autoclicker/dpi.py
|
|
16
17
|
autoclicker/expr.py
|
|
18
|
+
autoclicker/lint.py
|
|
17
19
|
autoclicker/matcher.py
|
|
18
20
|
autoclicker/models.py
|
|
19
21
|
autoclicker/ocr.py
|
|
20
22
|
autoclicker/run.py
|
|
21
23
|
autoclicker/runner.py
|
|
24
|
+
autoclicker/templatize.py
|
|
22
25
|
autoclicker/windows.py
|
|
23
26
|
autoclicker/gui/__init__.py
|
|
24
27
|
autoclicker/gui/flow_connection.py
|
|
25
28
|
autoclicker/gui/flow_editor.py
|
|
26
29
|
autoclicker/gui/flow_node.py
|
|
27
30
|
autoclicker/gui/hotkeys.py
|
|
31
|
+
autoclicker/gui/lint_dialog.py
|
|
28
32
|
autoclicker/gui/main_window.py
|
|
29
33
|
autoclicker/gui/monitor.py
|
|
30
34
|
autoclicker/gui/preview_label.py
|
|
@@ -53,10 +57,15 @@ autoclicker/web/static/app.js
|
|
|
53
57
|
autoclicker/web/static/index.html
|
|
54
58
|
autoclicker/web/static/style.css
|
|
55
59
|
tests/test_adb.py
|
|
60
|
+
tests/test_diagnose.py
|
|
61
|
+
tests/test_diagnose_gui.py
|
|
56
62
|
tests/test_flow_branch.py
|
|
63
|
+
tests/test_lint.py
|
|
57
64
|
tests/test_matcher.py
|
|
58
65
|
tests/test_models.py
|
|
66
|
+
tests/test_recorder_gui.py
|
|
59
67
|
tests/test_runner.py
|
|
68
|
+
tests/test_templatize.py
|
|
60
69
|
tests/test_testkit.py
|
|
61
70
|
tests/test_testkit_gui.py
|
|
62
71
|
tests/test_web.py
|
|
@@ -28,6 +28,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
|
|
|
28
28
|
autoclicker # desktop GUI (flow editor / list editor / test tab)
|
|
29
29
|
autoclicker-run task.json --window "Notepad" # headless run against a window
|
|
30
30
|
autoclicker-run task.json --adb # run against an Android device (adb)
|
|
31
|
+
autoclicker-lint task.json # health-check a task (exit code = error count)
|
|
31
32
|
autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
|
|
32
33
|
autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
|
|
33
34
|
```
|
|
@@ -44,7 +45,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
44
45
|
- **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
|
|
45
46
|
changing — replaces guessed `sleep`s (fast when ready, patient when slow).
|
|
46
47
|
- **~30+ step types**: click image/text/position, click-all, relative/offset
|
|
47
|
-
clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
|
|
48
|
+
clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
|
|
48
49
|
key / type / hotkey, loop / goto / while / do-until, if image/text/color/
|
|
49
50
|
count/var with **true/false branching** (if-else), set-var, read-text→variable,
|
|
50
51
|
activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
|
|
@@ -55,12 +56,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
|
|
|
55
56
|
Drag-and-drop **node flow graph** (condition nodes have **true/false branch
|
|
56
57
|
ports** for visual if-else; a terminal **stop** node ends the flow) and an
|
|
57
58
|
interoperable **step list** with a full
|
|
58
|
-
per-step dialog;
|
|
59
|
+
per-step dialog; snippet library, task
|
|
59
60
|
packages (`.zip` with templates), on-preview coordinate/color picking, template
|
|
60
61
|
crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
|
|
61
62
|
and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
|
|
62
63
|
run visual test suites.
|
|
63
64
|
|
|
65
|
+
### Smart recorder
|
|
66
|
+
|
|
67
|
+
The action **recorder** (global hooks) doesn't just replay coordinates and
|
|
68
|
+
sleeps — with **智能录制 / smart mode** on (default) it records what things *look
|
|
69
|
+
like* instead of where they *were*:
|
|
70
|
+
|
|
71
|
+
- **Clicks become image steps.** A patch is cropped around the click and grown
|
|
72
|
+
(48 → 192 px) until it is both textured enough and **unique on screen**, then
|
|
73
|
+
emitted as a `click_image` step — so the flow still works after the window
|
|
74
|
+
moves or the resolution changes. Identical patches are de-duplicated by
|
|
75
|
+
content hash.
|
|
76
|
+
- **Drags become `drag_image`.** The drag's *start* is anchored to an image and
|
|
77
|
+
the *displacement* is stored, so start and end move together.
|
|
78
|
+
- **Pauses become `wait_stable`.** A long pause is only converted if the
|
|
79
|
+
recorder actually **observed the screen settle** during it; on a permanently
|
|
80
|
+
animating screen it keeps the fixed wait (a `wait_stable` that never settles
|
|
81
|
+
would fail the run).
|
|
82
|
+
|
|
83
|
+
Anything that can't be made robust falls back to the plain coordinate/fixed-wait
|
|
84
|
+
step and says why in the log.
|
|
85
|
+
|
|
86
|
+
## Diagnostics
|
|
87
|
+
|
|
88
|
+
**Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
|
|
89
|
+
statically scans a task and separates *will definitely break* from *works but
|
|
90
|
+
fragile*:
|
|
91
|
+
|
|
92
|
+
- **errors** — missing template / sub-task file, image step with no template,
|
|
93
|
+
loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
|
|
94
|
+
for confirmation before running a task with errors; the CLI exit code is the
|
|
95
|
+
error count.
|
|
96
|
+
- **warnings** — steps that depend on absolute coordinates, long fixed waits,
|
|
97
|
+
a match threshold low enough to hit the wrong thing, templates too small to be
|
|
98
|
+
unique. These are exactly the things the smart recorder now avoids, so the
|
|
99
|
+
check doubles as a "what should I re-record?" list.
|
|
100
|
+
|
|
101
|
+
**Failure attribution.** A run captures a screenshot per step; the first
|
|
102
|
+
successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
|
|
103
|
+
run fails, the frames are compared step by step and the report names the step
|
|
104
|
+
where the screen **first diverged** from the successful run — which is usually
|
|
105
|
+
*earlier* than the step that failed (step 7 can't find its button because a popup
|
|
106
|
+
appeared at step 4). You get the diverging step, the per-step changed-pixel
|
|
107
|
+
ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
|
|
108
|
+
that points at timing/flakiness rather than a UI change.
|
|
109
|
+
|
|
64
110
|
## Visual test framework (`autoclicker.testkit`)
|
|
65
111
|
|
|
66
112
|
Turn automation into tests:
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""失败归因:拿上一次成功运行的逐步截图当基准,指出画面**第一次**偏离发生在哪一步。
|
|
2
|
+
|
|
3
|
+
动机:失败的步骤往往不是出问题的步骤。第 7 步找不到图,真正的原因可能是第 4 步弹出了
|
|
4
|
+
公告框。日志只会告诉你"第 7 步失败",于是排查全靠人肉翻截图。
|
|
5
|
+
|
|
6
|
+
timeline 帧结构沿用 TaskRunner 的 `{"index": int, "desc": str, "png": bytes}`。
|
|
7
|
+
Qt-free。
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
|
|
16
|
+
import cv2
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
from autoclicker import matcher
|
|
20
|
+
|
|
21
|
+
BASELINE_ROOT = ".timelines"
|
|
22
|
+
DEFAULT_TOL = 0.02 # 变化像素比例超过这个就算"画面不一样了"
|
|
23
|
+
INDEX_FILE = "index.json"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class StepDiff:
|
|
28
|
+
index: int
|
|
29
|
+
desc: str
|
|
30
|
+
ratio: float # 变化像素比例;1.0 也可能表示尺寸不一致
|
|
31
|
+
note: str = ""
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def diverged(self) -> bool:
|
|
35
|
+
return self.ratio > DEFAULT_TOL
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Attribution:
|
|
40
|
+
failed_index: int
|
|
41
|
+
diverged_index: int | None # 画面第一次偏离的步骤;None = 没查出偏离
|
|
42
|
+
diffs: list[StepDiff] = field(default_factory=list)
|
|
43
|
+
triptych_png: bytes | None = None # 偏离那一步的 基准|实际|热力图
|
|
44
|
+
message: str = ""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def safe_name(name: str) -> str:
|
|
48
|
+
s = re.sub(r'[<>:"/\\|?*\s]+', "_", (name or "task").strip())
|
|
49
|
+
return s[:60] or "task"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def baseline_dir(task_name: str, root: str = BASELINE_ROOT) -> str:
|
|
53
|
+
return os.path.join(root, safe_name(task_name))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ─────────────────── 持久化 ───────────────────
|
|
57
|
+
|
|
58
|
+
def save_timeline(timeline: list[dict], dirname: str) -> int:
|
|
59
|
+
os.makedirs(dirname, exist_ok=True)
|
|
60
|
+
meta = []
|
|
61
|
+
for pos, f in enumerate(timeline):
|
|
62
|
+
fname = f"frame_{pos:03d}_step_{int(f['index']):03d}.png"
|
|
63
|
+
with open(os.path.join(dirname, fname), "wb") as fh:
|
|
64
|
+
fh.write(f["png"])
|
|
65
|
+
meta.append({"file": fname, "index": int(f["index"]), "desc": f.get("desc", "")})
|
|
66
|
+
with open(os.path.join(dirname, INDEX_FILE), "w", encoding="utf-8") as fh:
|
|
67
|
+
json.dump(meta, fh, ensure_ascii=False, indent=2)
|
|
68
|
+
return len(meta)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def load_timeline(dirname: str) -> list[dict]:
|
|
72
|
+
idx = os.path.join(dirname, INDEX_FILE)
|
|
73
|
+
if not os.path.isfile(idx):
|
|
74
|
+
return []
|
|
75
|
+
with open(idx, "r", encoding="utf-8") as fh:
|
|
76
|
+
meta = json.load(fh)
|
|
77
|
+
out = []
|
|
78
|
+
for m in meta:
|
|
79
|
+
p = os.path.join(dirname, m["file"])
|
|
80
|
+
if not os.path.isfile(p):
|
|
81
|
+
continue
|
|
82
|
+
with open(p, "rb") as fh:
|
|
83
|
+
out.append({"index": int(m["index"]), "desc": m.get("desc", ""), "png": fh.read()})
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def has_baseline(dirname: str) -> bool:
|
|
88
|
+
return os.path.isfile(os.path.join(dirname, INDEX_FILE))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def save_baseline_if_absent(timeline: list[dict], dirname: str) -> bool:
|
|
92
|
+
"""只在还没有基准时写。沿用 visual.compare_region 的"无基准就创建"惯例。
|
|
93
|
+
返回是否真的写了。"""
|
|
94
|
+
if not timeline or has_baseline(dirname):
|
|
95
|
+
return False
|
|
96
|
+
save_timeline(timeline, dirname)
|
|
97
|
+
return True
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ─────────────────── 比对 ───────────────────
|
|
101
|
+
|
|
102
|
+
def _decode(png: bytes):
|
|
103
|
+
img = cv2.imdecode(np.frombuffer(png, dtype=np.uint8), cv2.IMREAD_COLOR)
|
|
104
|
+
return None if img is None or img.size == 0 else img
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _by_step(timeline: list[dict]) -> dict[int, dict]:
|
|
108
|
+
"""按步骤下标取**首帧**。循环会让同一步出现多次,比对时只认第一次进入那一帧。"""
|
|
109
|
+
out: dict[int, dict] = {}
|
|
110
|
+
for f in timeline:
|
|
111
|
+
out.setdefault(int(f["index"]), f)
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def compare_timelines(baseline: list[dict], actual: list[dict],
|
|
116
|
+
tol: float = DEFAULT_TOL) -> list[StepDiff]:
|
|
117
|
+
"""按步骤下标对齐两条 timeline,逐步算变化像素比例。"""
|
|
118
|
+
base_map = _by_step(baseline)
|
|
119
|
+
diffs: list[StepDiff] = []
|
|
120
|
+
for f in _by_step(actual).values():
|
|
121
|
+
i = int(f["index"])
|
|
122
|
+
desc = f.get("desc", "")
|
|
123
|
+
b = base_map.get(i)
|
|
124
|
+
if b is None:
|
|
125
|
+
diffs.append(StepDiff(i, desc, 0.0, "基准里没有这一步(流程走了不同分支?)"))
|
|
126
|
+
continue
|
|
127
|
+
a_img, b_img = _decode(f["png"]), _decode(b["png"])
|
|
128
|
+
if a_img is None or b_img is None:
|
|
129
|
+
diffs.append(StepDiff(i, desc, 0.0, "截图无法解码"))
|
|
130
|
+
continue
|
|
131
|
+
if a_img.shape != b_img.shape:
|
|
132
|
+
diffs.append(StepDiff(i, desc, 1.0,
|
|
133
|
+
f"画面尺寸变了 基准{b_img.shape[1]}x{b_img.shape[0]} "
|
|
134
|
+
f"vs 实际{a_img.shape[1]}x{a_img.shape[0]}"))
|
|
135
|
+
continue
|
|
136
|
+
diffs.append(StepDiff(i, desc, matcher.diff_ratio(a_img, b_img)))
|
|
137
|
+
diffs.sort(key=lambda d: d.index)
|
|
138
|
+
return diffs
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _triptych_for(baseline: list[dict], actual: list[dict], index: int) -> bytes | None:
|
|
142
|
+
from autoclicker.testkit.visual import heatmap, triptych
|
|
143
|
+
b = _by_step(baseline).get(index)
|
|
144
|
+
a = _by_step(actual).get(index)
|
|
145
|
+
if b is None or a is None:
|
|
146
|
+
return None
|
|
147
|
+
b_img, a_img = _decode(b["png"]), _decode(a["png"])
|
|
148
|
+
if b_img is None or a_img is None:
|
|
149
|
+
return None
|
|
150
|
+
if b_img.shape != a_img.shape:
|
|
151
|
+
return triptych(b_img, a_img, np.zeros_like(a_img))
|
|
152
|
+
per_pixel_max = cv2.absdiff(b_img, a_img).max(axis=2)
|
|
153
|
+
return triptych(b_img, a_img, heatmap(per_pixel_max))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def attribute(baseline: list[dict], actual: list[dict], failed_index: int,
|
|
157
|
+
tol: float = DEFAULT_TOL) -> Attribution:
|
|
158
|
+
"""归因。重点在 diverged_index:画面第一次和成功那次不一样是在哪一步。"""
|
|
159
|
+
if not baseline:
|
|
160
|
+
return Attribution(failed_index, None, [], None,
|
|
161
|
+
"没有基准时间线可比对(成功跑一次后会自动建立)")
|
|
162
|
+
diffs = compare_timelines(baseline, actual, tol)
|
|
163
|
+
diverged = next((d for d in diffs if d.ratio > tol), None)
|
|
164
|
+
if diverged is None:
|
|
165
|
+
return Attribution(failed_index, None, diffs, None,
|
|
166
|
+
f"第 {failed_index + 1} 步失败,但逐步画面和成功那次基本一致"
|
|
167
|
+
f"(最大偏离 {max((d.ratio for d in diffs), default=0.0):.4f});"
|
|
168
|
+
f"可能是时序/随机性问题,而不是界面变了")
|
|
169
|
+
png = _triptych_for(baseline, actual, diverged.index)
|
|
170
|
+
if diverged.index < failed_index:
|
|
171
|
+
msg = (f"第 {failed_index + 1} 步失败,但画面从**第 {diverged.index + 1} 步**"
|
|
172
|
+
f"就开始和成功那次不一样了(偏离 {diverged.ratio:.4f})"
|
|
173
|
+
f"{' — ' + diverged.note if diverged.note else ''}。"
|
|
174
|
+
f"先查这一步: {diverged.desc}")
|
|
175
|
+
else:
|
|
176
|
+
msg = (f"第 {failed_index + 1} 步失败,画面偏离也正是从这一步开始"
|
|
177
|
+
f"(偏离 {diverged.ratio:.4f})"
|
|
178
|
+
f"{' — ' + diverged.note if diverged.note else ''}")
|
|
179
|
+
return Attribution(failed_index, diverged.index, diffs, png, msg)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""任务体检结果弹窗:按严重程度着色,双击一条直接跳到对应步骤。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from PySide6.QtCore import Qt, Signal
|
|
5
|
+
from PySide6.QtGui import QColor
|
|
6
|
+
from PySide6.QtWidgets import (
|
|
7
|
+
QDialog, QLabel, QListWidget, QListWidgetItem, QPushButton, QVBoxLayout,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
from autoclicker.lint import ERROR, INFO, WARN, Finding, counts
|
|
11
|
+
|
|
12
|
+
_COLORS = {ERROR: QColor("#c62828"), WARN: QColor("#ef6c00"), INFO: QColor("#546e7a")}
|
|
13
|
+
_ICONS = {ERROR: "✖", WARN: "▲", INFO: "·"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class LintDialog(QDialog):
|
|
17
|
+
stepActivated = Signal(int) # 双击某条 → 该跳到的步骤下标
|
|
18
|
+
|
|
19
|
+
def __init__(self, findings: list[Finding], step_count: int, parent=None):
|
|
20
|
+
super().__init__(parent)
|
|
21
|
+
self.setWindowTitle("任务体检")
|
|
22
|
+
self.resize(720, 420)
|
|
23
|
+
lay = QVBoxLayout(self)
|
|
24
|
+
|
|
25
|
+
c = counts(findings)
|
|
26
|
+
if not findings:
|
|
27
|
+
summary = f"没查出问题({step_count} 步)。"
|
|
28
|
+
else:
|
|
29
|
+
summary = (f"共 {step_count} 步: {c[ERROR]} 个错误(会导致跑错), "
|
|
30
|
+
f"{c[WARN]} 处脆弱(能跑但不稳), {c[INFO]} 条提示 —— 双击可跳到该步骤")
|
|
31
|
+
lay.addWidget(QLabel(summary))
|
|
32
|
+
|
|
33
|
+
self.list = QListWidget()
|
|
34
|
+
self.list.itemDoubleClicked.connect(self._on_double)
|
|
35
|
+
for level in (ERROR, WARN, INFO):
|
|
36
|
+
for f in findings:
|
|
37
|
+
if f.level != level:
|
|
38
|
+
continue
|
|
39
|
+
where = "整个任务" if f.index < 0 else f"步骤 {f.index + 1}"
|
|
40
|
+
it = QListWidgetItem(f"{_ICONS[level]} {where} {f.message}")
|
|
41
|
+
it.setForeground(_COLORS[level])
|
|
42
|
+
it.setData(Qt.UserRole, f.index)
|
|
43
|
+
self.list.addItem(it)
|
|
44
|
+
lay.addWidget(self.list, 1)
|
|
45
|
+
|
|
46
|
+
close = QPushButton("关闭")
|
|
47
|
+
close.clicked.connect(self.accept)
|
|
48
|
+
lay.addWidget(close)
|
|
49
|
+
|
|
50
|
+
def _on_double(self, item: QListWidgetItem):
|
|
51
|
+
index = item.data(Qt.UserRole)
|
|
52
|
+
if index is not None and index >= 0:
|
|
53
|
+
self.stepActivated.emit(int(index))
|