PyClickerRPA 0.3.1__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PKG-INFO +50 -4
  2. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/PKG-INFO +50 -4
  3. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/SOURCES.txt +9 -0
  4. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/entry_points.txt +1 -0
  5. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/README.md +48 -2
  6. pyclickerrpa-0.5.0/autoclicker/diagnose.py +179 -0
  7. pyclickerrpa-0.5.0/autoclicker/gui/lint_dialog.py +53 -0
  8. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/main_window.py +132 -6
  9. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/qt_runner.py +6 -1
  10. pyclickerrpa-0.5.0/autoclicker/gui/recorder.py +290 -0
  11. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/task_widget.py +13 -4
  12. pyclickerrpa-0.5.0/autoclicker/lint.py +163 -0
  13. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/models.py +3 -0
  14. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/runner.py +38 -0
  15. pyclickerrpa-0.5.0/autoclicker/templatize.py +119 -0
  16. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/visual.py +11 -4
  17. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/pyproject.toml +3 -2
  18. pyclickerrpa-0.5.0/tests/test_diagnose.py +198 -0
  19. pyclickerrpa-0.5.0/tests/test_diagnose_gui.py +97 -0
  20. pyclickerrpa-0.5.0/tests/test_lint.py +155 -0
  21. pyclickerrpa-0.5.0/tests/test_recorder_gui.py +321 -0
  22. pyclickerrpa-0.5.0/tests/test_templatize.py +133 -0
  23. pyclickerrpa-0.3.1/autoclicker/gui/recorder.py +0 -163
  24. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/LICENSE +0 -0
  25. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/dependency_links.txt +0 -0
  26. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/requires.txt +0 -0
  27. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/PyClickerRPA.egg-info/top_level.txt +0 -0
  28. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/__init__.py +0 -0
  29. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/adb.py +0 -0
  30. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/app.py +0 -0
  31. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/capture.py +0 -0
  32. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/clicker.py +0 -0
  33. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/dpi.py +0 -0
  34. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/expr.py +0 -0
  35. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/__init__.py +0 -0
  36. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_connection.py +0 -0
  37. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_editor.py +0 -0
  38. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/flow_node.py +0 -0
  39. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/hotkeys.py +0 -0
  40. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/monitor.py +0 -0
  41. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/preview_label.py +0 -0
  42. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/scheduler_widget.py +0 -0
  43. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/snippet_library.py +0 -0
  44. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/gui/testkit_widget.py +0 -0
  45. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/matcher.py +0 -0
  46. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/ocr.py +0 -0
  47. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/run.py +0 -0
  48. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/__init__.py +0 -0
  49. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/__main__.py +0 -0
  50. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/assertions.py +0 -0
  51. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/case.py +0 -0
  52. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/loader.py +0 -0
  53. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/pytest_plugin.py +0 -0
  54. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/report.py +0 -0
  55. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/testkit/runner.py +0 -0
  56. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/__init__.py +0 -0
  57. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/__main__.py +0 -0
  58. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/run_manager.py +0 -0
  59. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/scheduler.py +0 -0
  60. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/serialize.py +0 -0
  61. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/server.py +0 -0
  62. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/app.js +0 -0
  63. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/index.html +0 -0
  64. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/web/static/style.css +0 -0
  65. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/autoclicker/windows.py +0 -0
  66. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/setup.cfg +0 -0
  67. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_adb.py +0 -0
  68. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_flow_branch.py +0 -0
  69. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_matcher.py +0 -0
  70. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_models.py +0 -0
  71. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_runner.py +0 -0
  72. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_testkit.py +0 -0
  73. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_testkit_gui.py +0 -0
  74. {pyclickerrpa-0.3.1 → pyclickerrpa-0.5.0}/tests/test_web.py +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PyClickerRPA
3
- Version: 0.3.1
4
- Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, recorder, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
3
+ Version: 0.5.0
4
+ Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, a smart recorder that records images instead of coordinates, task health checks and failure attribution, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
5
5
  Author: Equinox
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/Equinox/PyClickerRPA
@@ -66,6 +66,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
66
66
  autoclicker # desktop GUI (flow editor / list editor / test tab)
67
67
  autoclicker-run task.json --window "Notepad" # headless run against a window
68
68
  autoclicker-run task.json --adb # run against an Android device (adb)
69
+ autoclicker-lint task.json # health-check a task (exit code = error count)
69
70
  autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
70
71
  autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
71
72
  ```
@@ -82,7 +83,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
82
83
  - **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
83
84
  changing — replaces guessed `sleep`s (fast when ready, patient when slow).
84
85
  - **~30+ step types**: click image/text/position, click-all, relative/offset
85
- clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
86
+ clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
86
87
  key / type / hotkey, loop / goto / while / do-until, if image/text/color/
87
88
  count/var with **true/false branching** (if-else), set-var, read-text→variable,
88
89
  activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
@@ -93,12 +94,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
93
94
  Drag-and-drop **node flow graph** (condition nodes have **true/false branch
94
95
  ports** for visual if-else; a terminal **stop** node ends the flow) and an
95
96
  interoperable **step list** with a full
96
- per-step dialog; an action **recorder** (global hooks), snippet library, task
97
+ per-step dialog; snippet library, task
97
98
  packages (`.zip` with templates), on-preview coordinate/color picking, template
98
99
  crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
99
100
  and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
100
101
  run visual test suites.
101
102
 
103
+ ### Smart recorder
104
+
105
+ The action **recorder** (global hooks) doesn't just replay coordinates and
106
+ sleeps — with **智能录制 / smart mode** on (default) it records what things *look
107
+ like* instead of where they *were*:
108
+
109
+ - **Clicks become image steps.** A patch is cropped around the click and grown
110
+ (48 → 192 px) until it is both textured enough and **unique on screen**, then
111
+ emitted as a `click_image` step — so the flow still works after the window
112
+ moves or the resolution changes. Identical patches are de-duplicated by
113
+ content hash.
114
+ - **Drags become `drag_image`.** The drag's *start* is anchored to an image and
115
+ the *displacement* is stored, so start and end move together.
116
+ - **Pauses become `wait_stable`.** A long pause is only converted if the
117
+ recorder actually **observed the screen settle** during it; on a permanently
118
+ animating screen it keeps the fixed wait (a `wait_stable` that never settles
119
+ would fail the run).
120
+
121
+ Anything that can't be made robust falls back to the plain coordinate/fixed-wait
122
+ step and says why in the log.
123
+
124
+ ## Diagnostics
125
+
126
+ **Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
127
+ statically scans a task and separates *will definitely break* from *works but
128
+ fragile*:
129
+
130
+ - **errors** — missing template / sub-task file, image step with no template,
131
+ loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
132
+ for confirmation before running a task with errors; the CLI exit code is the
133
+ error count.
134
+ - **warnings** — steps that depend on absolute coordinates, long fixed waits,
135
+ a match threshold low enough to hit the wrong thing, templates too small to be
136
+ unique. These are exactly the things the smart recorder now avoids, so the
137
+ check doubles as a "what should I re-record?" list.
138
+
139
+ **Failure attribution.** A run captures a screenshot per step; the first
140
+ successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
141
+ run fails, the frames are compared step by step and the report names the step
142
+ where the screen **first diverged** from the successful run — which is usually
143
+ *earlier* than the step that failed (step 7 can't find its button because a popup
144
+ appeared at step 4). You get the diverging step, the per-step changed-pixel
145
+ ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
146
+ that points at timing/flakiness rather than a UI change.
147
+
102
148
  ## Visual test framework (`autoclicker.testkit`)
103
149
 
104
150
  Turn automation into tests:
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: PyClickerRPA
3
- Version: 0.3.1
4
- Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, recorder, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
3
+ Version: 0.5.0
4
+ Summary: Visual UI automation & testing tool: Windows + Android(ADB) automation, robust/self-healing image matching, OCR, node flow editor, a smart recorder that records images instead of coordinates, task health checks and failure attribution, a visual test framework (assertions, visual regression, HTML/JUnit reports) and a web console.
5
5
  Author: Equinox
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/Equinox/PyClickerRPA
@@ -66,6 +66,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
66
66
  autoclicker # desktop GUI (flow editor / list editor / test tab)
67
67
  autoclicker-run task.json --window "Notepad" # headless run against a window
68
68
  autoclicker-run task.json --adb # run against an Android device (adb)
69
+ autoclicker-lint task.json # health-check a task (exit code = error count)
69
70
  autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
70
71
  autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
71
72
  ```
@@ -82,7 +83,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
82
83
  - **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
83
84
  changing — replaces guessed `sleep`s (fast when ready, patient when slow).
84
85
  - **~30+ step types**: click image/text/position, click-all, relative/offset
85
- clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
86
+ clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
86
87
  key / type / hotkey, loop / goto / while / do-until, if image/text/color/
87
88
  count/var with **true/false branching** (if-else), set-var, read-text→variable,
88
89
  activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
@@ -93,12 +94,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
93
94
  Drag-and-drop **node flow graph** (condition nodes have **true/false branch
94
95
  ports** for visual if-else; a terminal **stop** node ends the flow) and an
95
96
  interoperable **step list** with a full
96
- per-step dialog; an action **recorder** (global hooks), snippet library, task
97
+ per-step dialog; snippet library, task
97
98
  packages (`.zip` with templates), on-preview coordinate/color picking, template
98
99
  crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
99
100
  and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
100
101
  run visual test suites.
101
102
 
103
+ ### Smart recorder
104
+
105
+ The action **recorder** (global hooks) doesn't just replay coordinates and
106
+ sleeps — with **智能录制 / smart mode** on (default) it records what things *look
107
+ like* instead of where they *were*:
108
+
109
+ - **Clicks become image steps.** A patch is cropped around the click and grown
110
+ (48 → 192 px) until it is both textured enough and **unique on screen**, then
111
+ emitted as a `click_image` step — so the flow still works after the window
112
+ moves or the resolution changes. Identical patches are de-duplicated by
113
+ content hash.
114
+ - **Drags become `drag_image`.** The drag's *start* is anchored to an image and
115
+ the *displacement* is stored, so start and end move together.
116
+ - **Pauses become `wait_stable`.** A long pause is only converted if the
117
+ recorder actually **observed the screen settle** during it; on a permanently
118
+ animating screen it keeps the fixed wait (a `wait_stable` that never settles
119
+ would fail the run).
120
+
121
+ Anything that can't be made robust falls back to the plain coordinate/fixed-wait
122
+ step and says why in the log.
123
+
124
+ ## Diagnostics
125
+
126
+ **Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
127
+ statically scans a task and separates *will definitely break* from *works but
128
+ fragile*:
129
+
130
+ - **errors** — missing template / sub-task file, image step with no template,
131
+ loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
132
+ for confirmation before running a task with errors; the CLI exit code is the
133
+ error count.
134
+ - **warnings** — steps that depend on absolute coordinates, long fixed waits,
135
+ a match threshold low enough to hit the wrong thing, templates too small to be
136
+ unique. These are exactly the things the smart recorder now avoids, so the
137
+ check doubles as a "what should I re-record?" list.
138
+
139
+ **Failure attribution.** A run captures a screenshot per step; the first
140
+ successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
141
+ run fails, the frames are compared step by step and the report names the step
142
+ where the screen **first diverged** from the successful run — which is usually
143
+ *earlier* than the step that failed (step 7 can't find its button because a popup
144
+ appeared at step 4). You get the diverging step, the per-step changed-pixel
145
+ ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
146
+ that points at timing/flakiness rather than a UI change.
147
+
102
148
  ## Visual test framework (`autoclicker.testkit`)
103
149
 
104
150
  Turn automation into tests:
@@ -12,19 +12,23 @@ autoclicker/adb.py
12
12
  autoclicker/app.py
13
13
  autoclicker/capture.py
14
14
  autoclicker/clicker.py
15
+ autoclicker/diagnose.py
15
16
  autoclicker/dpi.py
16
17
  autoclicker/expr.py
18
+ autoclicker/lint.py
17
19
  autoclicker/matcher.py
18
20
  autoclicker/models.py
19
21
  autoclicker/ocr.py
20
22
  autoclicker/run.py
21
23
  autoclicker/runner.py
24
+ autoclicker/templatize.py
22
25
  autoclicker/windows.py
23
26
  autoclicker/gui/__init__.py
24
27
  autoclicker/gui/flow_connection.py
25
28
  autoclicker/gui/flow_editor.py
26
29
  autoclicker/gui/flow_node.py
27
30
  autoclicker/gui/hotkeys.py
31
+ autoclicker/gui/lint_dialog.py
28
32
  autoclicker/gui/main_window.py
29
33
  autoclicker/gui/monitor.py
30
34
  autoclicker/gui/preview_label.py
@@ -53,10 +57,15 @@ autoclicker/web/static/app.js
53
57
  autoclicker/web/static/index.html
54
58
  autoclicker/web/static/style.css
55
59
  tests/test_adb.py
60
+ tests/test_diagnose.py
61
+ tests/test_diagnose_gui.py
56
62
  tests/test_flow_branch.py
63
+ tests/test_lint.py
57
64
  tests/test_matcher.py
58
65
  tests/test_models.py
66
+ tests/test_recorder_gui.py
59
67
  tests/test_runner.py
68
+ tests/test_templatize.py
60
69
  tests/test_testkit.py
61
70
  tests/test_testkit_gui.py
62
71
  tests/test_web.py
@@ -1,4 +1,5 @@
1
1
  [console_scripts]
2
+ autoclicker-lint = autoclicker.lint:main
2
3
  autoclicker-run = autoclicker.run:main
3
4
  autoclicker-testkit = autoclicker.testkit.__main__:main
4
5
  autoclicker-web = autoclicker.web.__main__:main
@@ -28,6 +28,7 @@ pip install "PyClickerRPA[web]" # FastAPI + uvicorn — web console
28
28
  autoclicker # desktop GUI (flow editor / list editor / test tab)
29
29
  autoclicker-run task.json --window "Notepad" # headless run against a window
30
30
  autoclicker-run task.json --adb # run against an Android device (adb)
31
+ autoclicker-lint task.json # health-check a task (exit code = error count)
31
32
  autoclicker-testkit suite.json --report out.html --junit out.xml # run a test suite
32
33
  autoclicker-web --suites-dir examples # web console at http://127.0.0.1:8000
33
34
  ```
@@ -44,7 +45,7 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
44
45
  - **Smart wait**: `wait_stable` waits until the screen (or an ROI) stops
45
46
  changing — replaces guessed `sleep`s (fast when ready, patient when slow).
46
47
  - **~30+ step types**: click image/text/position, click-all, relative/offset
47
- clicks, wait / random / wait-stable, drag / long-press / pinch / scroll,
48
+ clicks, wait / random / wait-stable, drag / **drag-from-image** / long-press / pinch / scroll,
48
49
  key / type / hotkey, loop / goto / while / do-until, if image/text/color/
49
50
  count/var with **true/false branching** (if-else), set-var, read-text→variable,
50
51
  activate-window, HTTP request, run-Python-script, call sub-task, groups, and a
@@ -55,12 +56,57 @@ autoclicker-web --suites-dir examples # web console at http://127.0.0.1
55
56
  Drag-and-drop **node flow graph** (condition nodes have **true/false branch
56
57
  ports** for visual if-else; a terminal **stop** node ends the flow) and an
57
58
  interoperable **step list** with a full
58
- per-step dialog; an action **recorder** (global hooks), snippet library, task
59
+ per-step dialog; snippet library, task
59
60
  packages (`.zip` with templates), on-preview coordinate/color picking, template
60
61
  crop, single-step test run; global hotkeys (F6 run/stop, ESC stop), scheduler,
61
62
  and an image/color **monitor/trigger** with tray icon; a **Test** tab to load and
62
63
  run visual test suites.
63
64
 
65
+ ### Smart recorder
66
+
67
+ The action **recorder** (global hooks) doesn't just replay coordinates and
68
+ sleeps — with **智能录制 / smart mode** on (default) it records what things *look
69
+ like* instead of where they *were*:
70
+
71
+ - **Clicks become image steps.** A patch is cropped around the click and grown
72
+ (48 → 192 px) until it is both textured enough and **unique on screen**, then
73
+ emitted as a `click_image` step — so the flow still works after the window
74
+ moves or the resolution changes. Identical patches are de-duplicated by
75
+ content hash.
76
+ - **Drags become `drag_image`.** The drag's *start* is anchored to an image and
77
+ the *displacement* is stored, so start and end move together.
78
+ - **Pauses become `wait_stable`.** A long pause is only converted if the
79
+ recorder actually **observed the screen settle** during it; on a permanently
80
+ animating screen it keeps the fixed wait (a `wait_stable` that never settles
81
+ would fail the run).
82
+
83
+ Anything that can't be made robust falls back to the plain coordinate/fixed-wait
84
+ step and says why in the log.
85
+
86
+ ## Diagnostics
87
+
88
+ **Task health check** (`autoclicker-lint task.json`, or 诊断 → 任务体检 in the GUI)
89
+ statically scans a task and separates *will definitely break* from *works but
90
+ fragile*:
91
+
92
+ - **errors** — missing template / sub-task file, image step with no template,
93
+ loop / goto / fail-jump target out of range, unbalanced groups. The GUI asks
94
+ for confirmation before running a task with errors; the CLI exit code is the
95
+ error count.
96
+ - **warnings** — steps that depend on absolute coordinates, long fixed waits,
97
+ a match threshold low enough to hit the wrong thing, templates too small to be
98
+ unique. These are exactly the things the smart recorder now avoids, so the
99
+ check doubles as a "what should I re-record?" list.
100
+
101
+ **Failure attribution.** A run captures a screenshot per step; the first
102
+ successful run becomes the baseline timeline (`.timelines/<task>/`). When a later
103
+ run fails, the frames are compared step by step and the report names the step
104
+ where the screen **first diverged** from the successful run — which is usually
105
+ *earlier* than the step that failed (step 7 can't find its button because a popup
106
+ appeared at step 4). You get the diverging step, the per-step changed-pixel
107
+ ratios, and a baseline|actual|heatmap image. If the screens match, it says so —
108
+ that points at timing/flakiness rather than a UI change.
109
+
64
110
  ## Visual test framework (`autoclicker.testkit`)
65
111
 
66
112
  Turn automation into tests:
@@ -0,0 +1,179 @@
1
+ """失败归因:拿上一次成功运行的逐步截图当基准,指出画面**第一次**偏离发生在哪一步。
2
+
3
+ 动机:失败的步骤往往不是出问题的步骤。第 7 步找不到图,真正的原因可能是第 4 步弹出了
4
+ 公告框。日志只会告诉你"第 7 步失败",于是排查全靠人肉翻截图。
5
+
6
+ timeline 帧结构沿用 TaskRunner 的 `{"index": int, "desc": str, "png": bytes}`。
7
+ Qt-free。
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import os
13
+ import re
14
+ from dataclasses import dataclass, field
15
+
16
+ import cv2
17
+ import numpy as np
18
+
19
+ from autoclicker import matcher
20
+
21
+ BASELINE_ROOT = ".timelines"
22
+ DEFAULT_TOL = 0.02 # 变化像素比例超过这个就算"画面不一样了"
23
+ INDEX_FILE = "index.json"
24
+
25
+
26
+ @dataclass
27
+ class StepDiff:
28
+ index: int
29
+ desc: str
30
+ ratio: float # 变化像素比例;1.0 也可能表示尺寸不一致
31
+ note: str = ""
32
+
33
+ @property
34
+ def diverged(self) -> bool:
35
+ return self.ratio > DEFAULT_TOL
36
+
37
+
38
+ @dataclass
39
+ class Attribution:
40
+ failed_index: int
41
+ diverged_index: int | None # 画面第一次偏离的步骤;None = 没查出偏离
42
+ diffs: list[StepDiff] = field(default_factory=list)
43
+ triptych_png: bytes | None = None # 偏离那一步的 基准|实际|热力图
44
+ message: str = ""
45
+
46
+
47
+ def safe_name(name: str) -> str:
48
+ s = re.sub(r'[<>:"/\\|?*\s]+', "_", (name or "task").strip())
49
+ return s[:60] or "task"
50
+
51
+
52
+ def baseline_dir(task_name: str, root: str = BASELINE_ROOT) -> str:
53
+ return os.path.join(root, safe_name(task_name))
54
+
55
+
56
+ # ─────────────────── 持久化 ───────────────────
57
+
58
+ def save_timeline(timeline: list[dict], dirname: str) -> int:
59
+ os.makedirs(dirname, exist_ok=True)
60
+ meta = []
61
+ for pos, f in enumerate(timeline):
62
+ fname = f"frame_{pos:03d}_step_{int(f['index']):03d}.png"
63
+ with open(os.path.join(dirname, fname), "wb") as fh:
64
+ fh.write(f["png"])
65
+ meta.append({"file": fname, "index": int(f["index"]), "desc": f.get("desc", "")})
66
+ with open(os.path.join(dirname, INDEX_FILE), "w", encoding="utf-8") as fh:
67
+ json.dump(meta, fh, ensure_ascii=False, indent=2)
68
+ return len(meta)
69
+
70
+
71
+ def load_timeline(dirname: str) -> list[dict]:
72
+ idx = os.path.join(dirname, INDEX_FILE)
73
+ if not os.path.isfile(idx):
74
+ return []
75
+ with open(idx, "r", encoding="utf-8") as fh:
76
+ meta = json.load(fh)
77
+ out = []
78
+ for m in meta:
79
+ p = os.path.join(dirname, m["file"])
80
+ if not os.path.isfile(p):
81
+ continue
82
+ with open(p, "rb") as fh:
83
+ out.append({"index": int(m["index"]), "desc": m.get("desc", ""), "png": fh.read()})
84
+ return out
85
+
86
+
87
+ def has_baseline(dirname: str) -> bool:
88
+ return os.path.isfile(os.path.join(dirname, INDEX_FILE))
89
+
90
+
91
+ def save_baseline_if_absent(timeline: list[dict], dirname: str) -> bool:
92
+ """只在还没有基准时写。沿用 visual.compare_region 的"无基准就创建"惯例。
93
+ 返回是否真的写了。"""
94
+ if not timeline or has_baseline(dirname):
95
+ return False
96
+ save_timeline(timeline, dirname)
97
+ return True
98
+
99
+
100
+ # ─────────────────── 比对 ───────────────────
101
+
102
+ def _decode(png: bytes):
103
+ img = cv2.imdecode(np.frombuffer(png, dtype=np.uint8), cv2.IMREAD_COLOR)
104
+ return None if img is None or img.size == 0 else img
105
+
106
+
107
+ def _by_step(timeline: list[dict]) -> dict[int, dict]:
108
+ """按步骤下标取**首帧**。循环会让同一步出现多次,比对时只认第一次进入那一帧。"""
109
+ out: dict[int, dict] = {}
110
+ for f in timeline:
111
+ out.setdefault(int(f["index"]), f)
112
+ return out
113
+
114
+
115
+ def compare_timelines(baseline: list[dict], actual: list[dict],
116
+ tol: float = DEFAULT_TOL) -> list[StepDiff]:
117
+ """按步骤下标对齐两条 timeline,逐步算变化像素比例。"""
118
+ base_map = _by_step(baseline)
119
+ diffs: list[StepDiff] = []
120
+ for f in _by_step(actual).values():
121
+ i = int(f["index"])
122
+ desc = f.get("desc", "")
123
+ b = base_map.get(i)
124
+ if b is None:
125
+ diffs.append(StepDiff(i, desc, 0.0, "基准里没有这一步(流程走了不同分支?)"))
126
+ continue
127
+ a_img, b_img = _decode(f["png"]), _decode(b["png"])
128
+ if a_img is None or b_img is None:
129
+ diffs.append(StepDiff(i, desc, 0.0, "截图无法解码"))
130
+ continue
131
+ if a_img.shape != b_img.shape:
132
+ diffs.append(StepDiff(i, desc, 1.0,
133
+ f"画面尺寸变了 基准{b_img.shape[1]}x{b_img.shape[0]} "
134
+ f"vs 实际{a_img.shape[1]}x{a_img.shape[0]}"))
135
+ continue
136
+ diffs.append(StepDiff(i, desc, matcher.diff_ratio(a_img, b_img)))
137
+ diffs.sort(key=lambda d: d.index)
138
+ return diffs
139
+
140
+
141
+ def _triptych_for(baseline: list[dict], actual: list[dict], index: int) -> bytes | None:
142
+ from autoclicker.testkit.visual import heatmap, triptych
143
+ b = _by_step(baseline).get(index)
144
+ a = _by_step(actual).get(index)
145
+ if b is None or a is None:
146
+ return None
147
+ b_img, a_img = _decode(b["png"]), _decode(a["png"])
148
+ if b_img is None or a_img is None:
149
+ return None
150
+ if b_img.shape != a_img.shape:
151
+ return triptych(b_img, a_img, np.zeros_like(a_img))
152
+ per_pixel_max = cv2.absdiff(b_img, a_img).max(axis=2)
153
+ return triptych(b_img, a_img, heatmap(per_pixel_max))
154
+
155
+
156
+ def attribute(baseline: list[dict], actual: list[dict], failed_index: int,
157
+ tol: float = DEFAULT_TOL) -> Attribution:
158
+ """归因。重点在 diverged_index:画面第一次和成功那次不一样是在哪一步。"""
159
+ if not baseline:
160
+ return Attribution(failed_index, None, [], None,
161
+ "没有基准时间线可比对(成功跑一次后会自动建立)")
162
+ diffs = compare_timelines(baseline, actual, tol)
163
+ diverged = next((d for d in diffs if d.ratio > tol), None)
164
+ if diverged is None:
165
+ return Attribution(failed_index, None, diffs, None,
166
+ f"第 {failed_index + 1} 步失败,但逐步画面和成功那次基本一致"
167
+ f"(最大偏离 {max((d.ratio for d in diffs), default=0.0):.4f});"
168
+ f"可能是时序/随机性问题,而不是界面变了")
169
+ png = _triptych_for(baseline, actual, diverged.index)
170
+ if diverged.index < failed_index:
171
+ msg = (f"第 {failed_index + 1} 步失败,但画面从**第 {diverged.index + 1} 步**"
172
+ f"就开始和成功那次不一样了(偏离 {diverged.ratio:.4f})"
173
+ f"{' — ' + diverged.note if diverged.note else ''}。"
174
+ f"先查这一步: {diverged.desc}")
175
+ else:
176
+ msg = (f"第 {failed_index + 1} 步失败,画面偏离也正是从这一步开始"
177
+ f"(偏离 {diverged.ratio:.4f})"
178
+ f"{' — ' + diverged.note if diverged.note else ''}")
179
+ return Attribution(failed_index, diverged.index, diffs, png, msg)
@@ -0,0 +1,53 @@
1
+ """任务体检结果弹窗:按严重程度着色,双击一条直接跳到对应步骤。"""
2
+ from __future__ import annotations
3
+
4
+ from PySide6.QtCore import Qt, Signal
5
+ from PySide6.QtGui import QColor
6
+ from PySide6.QtWidgets import (
7
+ QDialog, QLabel, QListWidget, QListWidgetItem, QPushButton, QVBoxLayout,
8
+ )
9
+
10
+ from autoclicker.lint import ERROR, INFO, WARN, Finding, counts
11
+
12
+ _COLORS = {ERROR: QColor("#c62828"), WARN: QColor("#ef6c00"), INFO: QColor("#546e7a")}
13
+ _ICONS = {ERROR: "✖", WARN: "▲", INFO: "·"}
14
+
15
+
16
+ class LintDialog(QDialog):
17
+ stepActivated = Signal(int) # 双击某条 → 该跳到的步骤下标
18
+
19
+ def __init__(self, findings: list[Finding], step_count: int, parent=None):
20
+ super().__init__(parent)
21
+ self.setWindowTitle("任务体检")
22
+ self.resize(720, 420)
23
+ lay = QVBoxLayout(self)
24
+
25
+ c = counts(findings)
26
+ if not findings:
27
+ summary = f"没查出问题({step_count} 步)。"
28
+ else:
29
+ summary = (f"共 {step_count} 步: {c[ERROR]} 个错误(会导致跑错), "
30
+ f"{c[WARN]} 处脆弱(能跑但不稳), {c[INFO]} 条提示 —— 双击可跳到该步骤")
31
+ lay.addWidget(QLabel(summary))
32
+
33
+ self.list = QListWidget()
34
+ self.list.itemDoubleClicked.connect(self._on_double)
35
+ for level in (ERROR, WARN, INFO):
36
+ for f in findings:
37
+ if f.level != level:
38
+ continue
39
+ where = "整个任务" if f.index < 0 else f"步骤 {f.index + 1}"
40
+ it = QListWidgetItem(f"{_ICONS[level]} {where} {f.message}")
41
+ it.setForeground(_COLORS[level])
42
+ it.setData(Qt.UserRole, f.index)
43
+ self.list.addItem(it)
44
+ lay.addWidget(self.list, 1)
45
+
46
+ close = QPushButton("关闭")
47
+ close.clicked.connect(self.accept)
48
+ lay.addWidget(close)
49
+
50
+ def _on_double(self, item: QListWidgetItem):
51
+ index = item.data(Qt.UserRole)
52
+ if index is not None and index >= 0:
53
+ self.stepActivated.emit(int(index))