pyvision-bot 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyvision_bot/__init__.py +19 -0
- pyvision_bot/backends.py +236 -0
- pyvision_bot/bot.py +288 -0
- pyvision_bot/matcher.py +272 -0
- pyvision_bot/models.py +118 -0
- pyvision_bot/py.typed +0 -0
- pyvision_bot-0.1.0.dist-info/METADATA +256 -0
- pyvision_bot-0.1.0.dist-info/RECORD +11 -0
- pyvision_bot-0.1.0.dist-info/WHEEL +5 -0
- pyvision_bot-0.1.0.dist-info/licenses/LICENSE +21 -0
- pyvision_bot-0.1.0.dist-info/top_level.txt +1 -0
pyvision_bot/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""pyvision-bot: OpenCV image automation for desktop and Android (ADB)."""
|
|
2
|
+
|
|
3
|
+
from .backends import ADBBackend, BaseBackend, DesktopBackend
|
|
4
|
+
from .bot import VisionBot
|
|
5
|
+
from .matcher import TemplateMatcher
|
|
6
|
+
from .models import MatchResult, ROI
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"ADBBackend",
|
|
12
|
+
"BaseBackend",
|
|
13
|
+
"DesktopBackend",
|
|
14
|
+
"MatchResult",
|
|
15
|
+
"ROI",
|
|
16
|
+
"TemplateMatcher",
|
|
17
|
+
"VisionBot",
|
|
18
|
+
"__version__",
|
|
19
|
+
]
|
pyvision_bot/backends.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Screenshot and input backends for desktop and Android (ADB)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import subprocess
|
|
6
|
+
from abc import ABC, abstractmethod
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
import cv2
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseBackend(ABC):
|
|
14
|
+
"""Abstract screenshot and input backend.
|
|
15
|
+
|
|
16
|
+
Implementations must return screenshots as OpenCV-style BGR arrays.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def screenshot(self) -> np.ndarray:
|
|
21
|
+
"""Capture the current screen and return a BGR ``numpy`` array."""
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def click(self, x: int, y: int) -> None:
|
|
25
|
+
"""Click at ``(x, y)`` in screen coordinates."""
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def swipe(
|
|
29
|
+
self,
|
|
30
|
+
x1: int,
|
|
31
|
+
y1: int,
|
|
32
|
+
x2: int,
|
|
33
|
+
y2: int,
|
|
34
|
+
duration: float,
|
|
35
|
+
) -> None:
|
|
36
|
+
"""Drag from ``(x1, y1)`` to ``(x2, y2)`` over ``duration`` seconds."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class DesktopBackend(BaseBackend):
|
|
40
|
+
"""Desktop backend powered by ``mss`` and ``pyautogui``.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
monitor: Monitor index understood by ``mss``. ``1`` is the primary
|
|
44
|
+
monitor; ``0`` means the union of all monitors.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
ValueError: If ``monitor`` is negative.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(self, monitor: int = 1) -> None:
|
|
51
|
+
if monitor < 0:
|
|
52
|
+
raise ValueError("monitor must be zero or a positive index")
|
|
53
|
+
self.monitor = monitor
|
|
54
|
+
|
|
55
|
+
def screenshot(self) -> np.ndarray:
|
|
56
|
+
"""Capture one monitor and convert BGRA to OpenCV BGR."""
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
import mss
|
|
60
|
+
except ImportError as exc: # pragma: no cover - dependency is declared
|
|
61
|
+
raise RuntimeError(
|
|
62
|
+
"DesktopBackend requires the 'mss' package. "
|
|
63
|
+
"Install it with: pip install mss"
|
|
64
|
+
) from exc
|
|
65
|
+
|
|
66
|
+
with mss.mss() as capture:
|
|
67
|
+
try:
|
|
68
|
+
monitor = capture.monitors[self.monitor]
|
|
69
|
+
except IndexError as exc:
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"monitor {self.monitor} is not available; "
|
|
72
|
+
f"found {len(capture.monitors)} monitor entries"
|
|
73
|
+
) from exc
|
|
74
|
+
|
|
75
|
+
frame = np.asarray(capture.grab(monitor))
|
|
76
|
+
|
|
77
|
+
if frame.ndim == 3 and frame.shape[2] == 4:
|
|
78
|
+
return cv2.cvtColor(frame, cv2.COLOR_BGRA2BGR)
|
|
79
|
+
|
|
80
|
+
return frame
|
|
81
|
+
|
|
82
|
+
def click(self, x: int, y: int) -> None:
|
|
83
|
+
"""Click with the left mouse button."""
|
|
84
|
+
|
|
85
|
+
pyautogui = self._load_pyautogui()
|
|
86
|
+
pyautogui.click(x=int(x), y=int(y))
|
|
87
|
+
|
|
88
|
+
def swipe(
|
|
89
|
+
self,
|
|
90
|
+
x1: int,
|
|
91
|
+
y1: int,
|
|
92
|
+
x2: int,
|
|
93
|
+
y2: int,
|
|
94
|
+
duration: float,
|
|
95
|
+
) -> None:
|
|
96
|
+
"""Drag with the left mouse button held down."""
|
|
97
|
+
|
|
98
|
+
pyautogui = self._load_pyautogui()
|
|
99
|
+
pyautogui.moveTo(int(x1), int(y1))
|
|
100
|
+
pyautogui.dragTo(
|
|
101
|
+
int(x2),
|
|
102
|
+
int(y2),
|
|
103
|
+
duration=max(0.0, float(duration)),
|
|
104
|
+
button="left",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def _load_pyautogui():
|
|
109
|
+
try:
|
|
110
|
+
import pyautogui
|
|
111
|
+
except ImportError as exc: # pragma: no cover - dependency is declared
|
|
112
|
+
raise RuntimeError(
|
|
113
|
+
"DesktopBackend requires the 'pyautogui' package. "
|
|
114
|
+
"Install it with: pip install pyautogui"
|
|
115
|
+
) from exc
|
|
116
|
+
return pyautogui
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ADBBackend(BaseBackend):
|
|
120
|
+
"""Android backend that shells out to the ``adb`` executable.
|
|
121
|
+
|
|
122
|
+
Screenshots are decoded directly from the bytes returned by
|
|
123
|
+
``adb exec-out screencap -p``; no temporary file is written.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
device_serial: Optional device serial passed as ``adb -s``.
|
|
127
|
+
adb_path: Path or command name of the ADB executable.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
device_serial: Optional[str] = None,
|
|
133
|
+
adb_path: str = "adb",
|
|
134
|
+
) -> None:
|
|
135
|
+
self.device_serial = device_serial
|
|
136
|
+
self.adb_path = adb_path
|
|
137
|
+
|
|
138
|
+
def screenshot(self) -> np.ndarray:
|
|
139
|
+
"""Capture the device screen and decode the PNG in memory."""
|
|
140
|
+
|
|
141
|
+
data = self._run_bytes("exec-out", "screencap", "-p")
|
|
142
|
+
|
|
143
|
+
# Some ADB/Windows combinations translate LF to CRLF in the PNG
|
|
144
|
+
# stream. Only repair the data when the PNG signature is not intact.
|
|
145
|
+
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
146
|
+
data = data.replace(b"\r\n", b"\n")
|
|
147
|
+
|
|
148
|
+
buffer = np.frombuffer(data, dtype=np.uint8)
|
|
149
|
+
image = cv2.imdecode(buffer, cv2.IMREAD_COLOR)
|
|
150
|
+
if image is None:
|
|
151
|
+
raise RuntimeError("adb screencap did not return a valid PNG image")
|
|
152
|
+
|
|
153
|
+
return image
|
|
154
|
+
|
|
155
|
+
def click(self, x: int, y: int) -> None:
|
|
156
|
+
"""Send ``adb shell input tap``."""
|
|
157
|
+
|
|
158
|
+
self._run_text("shell", "input", "tap", str(int(x)), str(int(y)))
|
|
159
|
+
|
|
160
|
+
def swipe(
|
|
161
|
+
self,
|
|
162
|
+
x1: int,
|
|
163
|
+
y1: int,
|
|
164
|
+
x2: int,
|
|
165
|
+
y2: int,
|
|
166
|
+
duration: float,
|
|
167
|
+
) -> None:
|
|
168
|
+
"""Send ``adb shell input swipe`` with a millisecond duration."""
|
|
169
|
+
|
|
170
|
+
duration_ms = max(0, int(round(float(duration) * 1000)))
|
|
171
|
+
self._run_text(
|
|
172
|
+
"shell",
|
|
173
|
+
"input",
|
|
174
|
+
"swipe",
|
|
175
|
+
str(int(x1)),
|
|
176
|
+
str(int(y1)),
|
|
177
|
+
str(int(x2)),
|
|
178
|
+
str(int(y2)),
|
|
179
|
+
str(duration_ms),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def _base_command(self) -> List[str]:
|
|
183
|
+
command = [self.adb_path]
|
|
184
|
+
if self.device_serial:
|
|
185
|
+
command.extend(["-s", self.device_serial])
|
|
186
|
+
return command
|
|
187
|
+
|
|
188
|
+
def _run_text(self, *arguments: str) -> str:
|
|
189
|
+
command = self._base_command() + list(arguments)
|
|
190
|
+
try:
|
|
191
|
+
process = subprocess.run(
|
|
192
|
+
command,
|
|
193
|
+
capture_output=True,
|
|
194
|
+
text=True,
|
|
195
|
+
encoding="utf-8",
|
|
196
|
+
errors="replace",
|
|
197
|
+
check=False,
|
|
198
|
+
)
|
|
199
|
+
except FileNotFoundError as exc:
|
|
200
|
+
raise RuntimeError(
|
|
201
|
+
f"adb executable not found: {self.adb_path}. "
|
|
202
|
+
"Install Android platform-tools and add adb to PATH."
|
|
203
|
+
) from exc
|
|
204
|
+
|
|
205
|
+
if process.returncode != 0:
|
|
206
|
+
raise RuntimeError(
|
|
207
|
+
f"adb command failed ({process.returncode}): "
|
|
208
|
+
f"{' '.join(command)}\n{process.stderr.strip()}"
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
return process.stdout
|
|
212
|
+
|
|
213
|
+
def _run_bytes(self, *arguments: str) -> bytes:
|
|
214
|
+
command = self._base_command() + list(arguments)
|
|
215
|
+
try:
|
|
216
|
+
process = subprocess.run(
|
|
217
|
+
command,
|
|
218
|
+
capture_output=True,
|
|
219
|
+
check=False,
|
|
220
|
+
)
|
|
221
|
+
except FileNotFoundError as exc:
|
|
222
|
+
raise RuntimeError(
|
|
223
|
+
f"adb executable not found: {self.adb_path}. "
|
|
224
|
+
"Install Android platform-tools and add adb to PATH."
|
|
225
|
+
) from exc
|
|
226
|
+
|
|
227
|
+
if process.returncode != 0:
|
|
228
|
+
stderr = process.stderr
|
|
229
|
+
if isinstance(stderr, bytes):
|
|
230
|
+
stderr = stderr.decode("utf-8", errors="replace")
|
|
231
|
+
raise RuntimeError(
|
|
232
|
+
f"adb command failed ({process.returncode}): "
|
|
233
|
+
f"{' '.join(command)}\n{stderr.strip()}"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return process.stdout
|
pyvision_bot/bot.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""High-level image automation API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import random
|
|
6
|
+
import time
|
|
7
|
+
from typing import List, Literal, Optional, Tuple
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from .backends import ADBBackend, BaseBackend, DesktopBackend
|
|
12
|
+
from .matcher import ImageSource, TemplateMatcher
|
|
13
|
+
from .models import MatchResult, ROIType
|
|
14
|
+
|
|
15
|
+
Mode = Literal["desktop", "adb"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class VisionBot:
|
|
19
|
+
"""Image-driven automation for a desktop screen or an Android device.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
mode: ``"desktop"`` for ``mss`` + ``pyautogui``, or ``"adb"`` for an
|
|
23
|
+
Android device.
|
|
24
|
+
device_serial: Optional ADB device serial, for example
|
|
25
|
+
``"emulator-5554"``. Ignored in desktop mode.
|
|
26
|
+
default_threshold: Default template matching confidence in ``[0, 1]``.
|
|
27
|
+
backend: Optional pre-built backend. Useful for tests or custom
|
|
28
|
+
transports. When omitted, a backend is created from ``mode``.
|
|
29
|
+
matcher: Optional :class:`TemplateMatcher` instance.
|
|
30
|
+
rng: Optional random source used for jitter and human-like delays.
|
|
31
|
+
|
|
32
|
+
Raises:
|
|
33
|
+
ValueError: If ``mode`` is unknown or the threshold is invalid.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
#: Minimum and maximum delay inserted before a click or swipe.
|
|
37
|
+
MIN_ACTION_DELAY = 0.02
|
|
38
|
+
MAX_ACTION_DELAY = 0.08
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
mode: Mode = "desktop",
|
|
43
|
+
device_serial: Optional[str] = None,
|
|
44
|
+
default_threshold: float = 0.85,
|
|
45
|
+
*,
|
|
46
|
+
backend: Optional[BaseBackend] = None,
|
|
47
|
+
matcher: Optional[TemplateMatcher] = None,
|
|
48
|
+
rng: Optional[random.Random] = None,
|
|
49
|
+
) -> None:
|
|
50
|
+
if mode not in ("desktop", "adb"):
|
|
51
|
+
raise ValueError("mode must be 'desktop' or 'adb'")
|
|
52
|
+
if not 0.0 <= default_threshold <= 1.0:
|
|
53
|
+
raise ValueError("default_threshold must be between 0 and 1")
|
|
54
|
+
|
|
55
|
+
self.mode: Mode = mode
|
|
56
|
+
self.device_serial = device_serial
|
|
57
|
+
self.default_threshold = float(default_threshold)
|
|
58
|
+
self._rng = rng or random.Random()
|
|
59
|
+
|
|
60
|
+
if backend is not None:
|
|
61
|
+
self._backend = backend
|
|
62
|
+
elif mode == "desktop":
|
|
63
|
+
self._backend = DesktopBackend()
|
|
64
|
+
else:
|
|
65
|
+
self._backend = ADBBackend(device_serial=device_serial)
|
|
66
|
+
|
|
67
|
+
self._matcher = matcher or TemplateMatcher(threshold=self.default_threshold)
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def backend(self) -> BaseBackend:
|
|
71
|
+
"""Return the active backend."""
|
|
72
|
+
|
|
73
|
+
return self._backend
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def matcher(self) -> TemplateMatcher:
|
|
77
|
+
"""Return the active template matcher."""
|
|
78
|
+
|
|
79
|
+
return self._matcher
|
|
80
|
+
|
|
81
|
+
def screenshot(self) -> np.ndarray:
|
|
82
|
+
"""Capture the current screen as a BGR ``numpy`` array."""
|
|
83
|
+
|
|
84
|
+
image = self._backend.screenshot()
|
|
85
|
+
if not isinstance(image, np.ndarray) or image.size == 0:
|
|
86
|
+
raise RuntimeError("backend returned an empty screenshot")
|
|
87
|
+
return image
|
|
88
|
+
|
|
89
|
+
def find(
|
|
90
|
+
self,
|
|
91
|
+
template_path: ImageSource,
|
|
92
|
+
threshold: Optional[float] = None,
|
|
93
|
+
roi: Optional[ROIType] = None,
|
|
94
|
+
) -> Optional[MatchResult]:
|
|
95
|
+
"""Take a screenshot and return the best template match.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
template_path: Path to the template image, or an in-memory image.
|
|
99
|
+
threshold: Optional confidence override.
|
|
100
|
+
roi: Optional search region in absolute screen coordinates.
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
The best :class:`MatchResult`, or ``None`` when not found.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
threshold_value = self._resolve_threshold(threshold)
|
|
107
|
+
return self._matcher.find(
|
|
108
|
+
self.screenshot(),
|
|
109
|
+
template_path,
|
|
110
|
+
threshold=threshold_value,
|
|
111
|
+
roi=roi,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def find_all(
|
|
115
|
+
self,
|
|
116
|
+
template_path: ImageSource,
|
|
117
|
+
threshold: Optional[float] = None,
|
|
118
|
+
roi: Optional[ROIType] = None,
|
|
119
|
+
max_results: Optional[int] = None,
|
|
120
|
+
) -> List[MatchResult]:
|
|
121
|
+
"""Take a screenshot and return all non-overlapping matches."""
|
|
122
|
+
|
|
123
|
+
threshold_value = self._resolve_threshold(threshold)
|
|
124
|
+
return self._matcher.find_all(
|
|
125
|
+
self.screenshot(),
|
|
126
|
+
template_path,
|
|
127
|
+
threshold=threshold_value,
|
|
128
|
+
roi=roi,
|
|
129
|
+
max_results=max_results,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def click(self, x: int, y: int, jitter: int = 0) -> None:
|
|
133
|
+
"""Click at ``(x, y)`` with optional random pixel jitter.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
x: Target x coordinate.
|
|
137
|
+
y: Target y coordinate.
|
|
138
|
+
jitter: Maximum random offset in each axis, in pixels.
|
|
139
|
+
|
|
140
|
+
Raises:
|
|
141
|
+
ValueError: If ``jitter`` is negative.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
if jitter < 0:
|
|
145
|
+
raise ValueError("jitter must be zero or positive")
|
|
146
|
+
|
|
147
|
+
self._human_pause()
|
|
148
|
+
click_x, click_y = self._apply_jitter(int(x), int(y), int(jitter))
|
|
149
|
+
self._backend.click(click_x, click_y)
|
|
150
|
+
|
|
151
|
+
def swipe(
|
|
152
|
+
self,
|
|
153
|
+
x1: int,
|
|
154
|
+
y1: int,
|
|
155
|
+
x2: int,
|
|
156
|
+
y2: int,
|
|
157
|
+
duration: float = 0.3,
|
|
158
|
+
jitter: int = 0,
|
|
159
|
+
) -> None:
|
|
160
|
+
"""Drag between two points with optional jitter.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
x1: Start x coordinate.
|
|
164
|
+
y1: Start y coordinate.
|
|
165
|
+
x2: End x coordinate.
|
|
166
|
+
y2: End y coordinate.
|
|
167
|
+
duration: Gesture duration in seconds.
|
|
168
|
+
jitter: Maximum random offset in each axis, in pixels.
|
|
169
|
+
|
|
170
|
+
Raises:
|
|
171
|
+
ValueError: If ``duration`` or ``jitter`` is negative.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
if duration < 0:
|
|
175
|
+
raise ValueError("duration must be zero or positive")
|
|
176
|
+
if jitter < 0:
|
|
177
|
+
raise ValueError("jitter must be zero or positive")
|
|
178
|
+
|
|
179
|
+
self._human_pause()
|
|
180
|
+
start_x, start_y = self._apply_jitter(int(x1), int(y1), int(jitter))
|
|
181
|
+
end_x, end_y = self._apply_jitter(int(x2), int(y2), int(jitter))
|
|
182
|
+
self._backend.swipe(
|
|
183
|
+
start_x,
|
|
184
|
+
start_y,
|
|
185
|
+
end_x,
|
|
186
|
+
end_y,
|
|
187
|
+
float(duration),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def wait_for(
|
|
191
|
+
self,
|
|
192
|
+
template_path: ImageSource,
|
|
193
|
+
timeout: float = 10.0,
|
|
194
|
+
interval: float = 0.5,
|
|
195
|
+
threshold: Optional[float] = None,
|
|
196
|
+
roi: Optional[ROIType] = None,
|
|
197
|
+
) -> Optional[MatchResult]:
|
|
198
|
+
"""Poll the screen until a template appears or the timeout expires.
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
template_path: Path to the template image, or an in-memory image.
|
|
202
|
+
timeout: Maximum wait time in seconds.
|
|
203
|
+
interval: Delay between screenshots in seconds.
|
|
204
|
+
threshold: Optional confidence override.
|
|
205
|
+
roi: Optional search region in absolute screen coordinates.
|
|
206
|
+
|
|
207
|
+
Returns:
|
|
208
|
+
The first matching :class:`MatchResult`, or ``None`` on timeout.
|
|
209
|
+
|
|
210
|
+
Raises:
|
|
211
|
+
ValueError: If ``timeout`` is negative or ``interval`` is not
|
|
212
|
+
positive.
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
if timeout < 0:
|
|
216
|
+
raise ValueError("timeout must be zero or positive")
|
|
217
|
+
if interval <= 0:
|
|
218
|
+
raise ValueError("interval must be positive")
|
|
219
|
+
|
|
220
|
+
deadline = time.monotonic() + float(timeout)
|
|
221
|
+
|
|
222
|
+
while True:
|
|
223
|
+
match = self.find(template_path, threshold=threshold, roi=roi)
|
|
224
|
+
if match is not None:
|
|
225
|
+
return match
|
|
226
|
+
|
|
227
|
+
remaining = deadline - time.monotonic()
|
|
228
|
+
if remaining <= 0:
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
time.sleep(min(float(interval), remaining))
|
|
232
|
+
|
|
233
|
+
def wait_and_click(
|
|
234
|
+
self,
|
|
235
|
+
template_path: ImageSource,
|
|
236
|
+
timeout: float = 10.0,
|
|
237
|
+
interval: float = 0.5,
|
|
238
|
+
threshold: Optional[float] = None,
|
|
239
|
+
roi: Optional[ROIType] = None,
|
|
240
|
+
jitter: int = 3,
|
|
241
|
+
) -> bool:
|
|
242
|
+
"""Wait for a template, then click its center.
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
template_path: Path to the template image, or an in-memory image.
|
|
246
|
+
timeout: Maximum wait time in seconds.
|
|
247
|
+
interval: Delay between screenshots in seconds.
|
|
248
|
+
threshold: Optional confidence override.
|
|
249
|
+
roi: Optional search region in absolute screen coordinates.
|
|
250
|
+
jitter: Maximum random click offset in each axis, in pixels.
|
|
251
|
+
|
|
252
|
+
Returns:
|
|
253
|
+
``True`` when the template was found and clicked, otherwise
|
|
254
|
+
``False``.
|
|
255
|
+
"""
|
|
256
|
+
|
|
257
|
+
match = self.wait_for(
|
|
258
|
+
template_path,
|
|
259
|
+
timeout=timeout,
|
|
260
|
+
interval=interval,
|
|
261
|
+
threshold=threshold,
|
|
262
|
+
roi=roi,
|
|
263
|
+
)
|
|
264
|
+
if match is None:
|
|
265
|
+
return False
|
|
266
|
+
|
|
267
|
+
center_x, center_y = match.center
|
|
268
|
+
self.click(center_x, center_y, jitter=jitter)
|
|
269
|
+
return True
|
|
270
|
+
|
|
271
|
+
def _resolve_threshold(self, threshold: Optional[float]) -> float:
|
|
272
|
+
value = self.default_threshold if threshold is None else float(threshold)
|
|
273
|
+
if not 0.0 <= value <= 1.0:
|
|
274
|
+
raise ValueError("threshold must be between 0 and 1")
|
|
275
|
+
return value
|
|
276
|
+
|
|
277
|
+
def _apply_jitter(self, x: int, y: int, jitter: int) -> Tuple[int, int]:
|
|
278
|
+
if jitter <= 0:
|
|
279
|
+
return x, y
|
|
280
|
+
return (
|
|
281
|
+
x + self._rng.randint(-jitter, jitter),
|
|
282
|
+
y + self._rng.randint(-jitter, jitter),
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def _human_pause(self) -> None:
|
|
286
|
+
time.sleep(
|
|
287
|
+
self._rng.uniform(self.MIN_ACTION_DELAY, self.MAX_ACTION_DELAY)
|
|
288
|
+
)
|
pyvision_bot/matcher.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""OpenCV template matching with ROI support and non-maximum suppression."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import List, Optional, Tuple, Union
|
|
7
|
+
|
|
8
|
+
import cv2
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from .models import MatchResult, ROIType, normalize_roi
|
|
12
|
+
|
|
13
|
+
ImageSource = Union[str, Path, np.ndarray]
|
|
14
|
+
PreparedMatch = Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TemplateMatcher:
|
|
18
|
+
"""Find template images inside a screenshot.
|
|
19
|
+
|
|
20
|
+
The matcher uses ``cv2.matchTemplate`` with ``cv2.TM_CCOEFF_NORMED``.
|
|
21
|
+
A constant (single-color) template has no correlation variance, so the
|
|
22
|
+
matcher automatically falls back to ``cv2.TM_SQDIFF_NORMED`` for that
|
|
23
|
+
case and converts the score to the same "higher is better" scale.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
threshold: Default confidence threshold in ``[0, 1]``.
|
|
27
|
+
grayscale: Convert both images to grayscale before matching. This is
|
|
28
|
+
faster and usually accurate enough for UI automation.
|
|
29
|
+
nms_iou: Intersection-over-union above which :meth:`find_all`
|
|
30
|
+
suppresses a lower-confidence box.
|
|
31
|
+
|
|
32
|
+
Raises:
|
|
33
|
+
ValueError: If a threshold or NMS value is outside ``[0, 1]``.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
threshold: float = 0.85,
|
|
39
|
+
grayscale: bool = True,
|
|
40
|
+
nms_iou: float = 0.3,
|
|
41
|
+
) -> None:
|
|
42
|
+
if not 0.0 <= threshold <= 1.0:
|
|
43
|
+
raise ValueError("threshold must be between 0 and 1")
|
|
44
|
+
if not 0.0 <= nms_iou <= 1.0:
|
|
45
|
+
raise ValueError("nms_iou must be between 0 and 1")
|
|
46
|
+
|
|
47
|
+
self.threshold = float(threshold)
|
|
48
|
+
self.grayscale = bool(grayscale)
|
|
49
|
+
self.nms_iou = float(nms_iou)
|
|
50
|
+
|
|
51
|
+
def find(
|
|
52
|
+
self,
|
|
53
|
+
screen: np.ndarray,
|
|
54
|
+
template_path: ImageSource,
|
|
55
|
+
threshold: Optional[float] = None,
|
|
56
|
+
roi: Optional[ROIType] = None,
|
|
57
|
+
) -> Optional[MatchResult]:
|
|
58
|
+
"""Return the best match, or ``None`` when nothing reaches the threshold.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
screen: Screenshot as a BGR (or grayscale) ``numpy`` array.
|
|
62
|
+
template_path: Path to the template image, or an in-memory image.
|
|
63
|
+
threshold: Override the matcher's default threshold.
|
|
64
|
+
roi: Optional search region. Returned coordinates are always
|
|
65
|
+
absolute screen coordinates.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
The best :class:`MatchResult`, or ``None``.
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
ValueError: If the screenshot is empty, the ROI is invalid, or the
|
|
72
|
+
threshold is outside ``[0, 1]``.
|
|
73
|
+
FileNotFoundError: If the template path cannot be read.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
threshold_value = self._resolve_threshold(threshold)
|
|
77
|
+
prepared = self._prepare(screen, template_path, roi)
|
|
78
|
+
if prepared is None:
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
scores, offset, (template_width, template_height) = prepared
|
|
82
|
+
_, max_value, _, max_location = cv2.minMaxLoc(scores)
|
|
83
|
+
if max_value < threshold_value:
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
return MatchResult(
|
|
87
|
+
x=int(max_location[0]) + offset[0],
|
|
88
|
+
y=int(max_location[1]) + offset[1],
|
|
89
|
+
width=template_width,
|
|
90
|
+
height=template_height,
|
|
91
|
+
confidence=float(max_value),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def find_all(
|
|
95
|
+
self,
|
|
96
|
+
screen: np.ndarray,
|
|
97
|
+
template_path: ImageSource,
|
|
98
|
+
threshold: Optional[float] = None,
|
|
99
|
+
roi: Optional[ROIType] = None,
|
|
100
|
+
max_results: Optional[int] = None,
|
|
101
|
+
) -> List[MatchResult]:
|
|
102
|
+
"""Return all non-overlapping matches above the threshold.
|
|
103
|
+
|
|
104
|
+
Candidate locations are sorted by confidence and filtered with
|
|
105
|
+
non-maximum suppression (NMS), so adjacent peaks belonging to the same
|
|
106
|
+
target do not produce duplicate results.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
screen: Screenshot as a BGR (or grayscale) ``numpy`` array.
|
|
110
|
+
template_path: Path to the template image, or an in-memory image.
|
|
111
|
+
threshold: Override the matcher's default threshold.
|
|
112
|
+
roi: Optional search region. Returned coordinates are always
|
|
113
|
+
absolute screen coordinates.
|
|
114
|
+
max_results: Optional cap on the number of returned matches.
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
Matches sorted by descending confidence. Empty when nothing is
|
|
118
|
+
found or the template is larger than the search area.
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
ValueError: If ``max_results`` is not positive, the screenshot is
|
|
122
|
+
empty, the ROI is invalid, or the threshold is invalid.
|
|
123
|
+
FileNotFoundError: If the template path cannot be read.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
if max_results is not None and max_results <= 0:
|
|
127
|
+
raise ValueError("max_results must be positive")
|
|
128
|
+
|
|
129
|
+
threshold_value = self._resolve_threshold(threshold)
|
|
130
|
+
prepared = self._prepare(screen, template_path, roi)
|
|
131
|
+
if prepared is None:
|
|
132
|
+
return []
|
|
133
|
+
|
|
134
|
+
scores, offset, (template_width, template_height) = prepared
|
|
135
|
+
ys, xs = np.where(scores >= threshold_value)
|
|
136
|
+
|
|
137
|
+
candidates = [
|
|
138
|
+
MatchResult(
|
|
139
|
+
x=int(x) + offset[0],
|
|
140
|
+
y=int(y) + offset[1],
|
|
141
|
+
width=template_width,
|
|
142
|
+
height=template_height,
|
|
143
|
+
confidence=float(scores[y, x]),
|
|
144
|
+
)
|
|
145
|
+
for y, x in zip(ys, xs)
|
|
146
|
+
]
|
|
147
|
+
candidates.sort(key=lambda item: item.confidence, reverse=True)
|
|
148
|
+
|
|
149
|
+
kept: List[MatchResult] = []
|
|
150
|
+
for candidate in candidates:
|
|
151
|
+
if any(self._iou(candidate, existing) > self.nms_iou for existing in kept):
|
|
152
|
+
continue
|
|
153
|
+
kept.append(candidate)
|
|
154
|
+
if max_results is not None and len(kept) >= max_results:
|
|
155
|
+
break
|
|
156
|
+
|
|
157
|
+
return kept
|
|
158
|
+
|
|
159
|
+
def _resolve_threshold(self, threshold: Optional[float]) -> float:
|
|
160
|
+
value = self.threshold if threshold is None else float(threshold)
|
|
161
|
+
if not 0.0 <= value <= 1.0:
|
|
162
|
+
raise ValueError("threshold must be between 0 and 1")
|
|
163
|
+
return value
|
|
164
|
+
|
|
165
|
+
def _prepare(
|
|
166
|
+
self,
|
|
167
|
+
screen: np.ndarray,
|
|
168
|
+
template_path: ImageSource,
|
|
169
|
+
roi: Optional[ROIType],
|
|
170
|
+
) -> Optional[PreparedMatch]:
|
|
171
|
+
if not isinstance(screen, np.ndarray) or screen.size == 0:
|
|
172
|
+
raise ValueError("screen must be a non-empty numpy array")
|
|
173
|
+
|
|
174
|
+
template = self._load_template(template_path)
|
|
175
|
+
screen_image = self._prepare_image(screen, self.grayscale)
|
|
176
|
+
template_image = self._prepare_image(template, self.grayscale)
|
|
177
|
+
|
|
178
|
+
offset = (0, 0)
|
|
179
|
+
search = screen_image
|
|
180
|
+
|
|
181
|
+
if roi is not None:
|
|
182
|
+
region = normalize_roi(roi).clamp(
|
|
183
|
+
screen_image.shape[1], screen_image.shape[0]
|
|
184
|
+
)
|
|
185
|
+
x, y, w, h = region.box
|
|
186
|
+
search = screen_image[y : y + h, x : x + w]
|
|
187
|
+
offset = (x, y)
|
|
188
|
+
|
|
189
|
+
template_height, template_width = template_image.shape[:2]
|
|
190
|
+
search_height, search_width = search.shape[:2]
|
|
191
|
+
|
|
192
|
+
# A template larger than the search area can never match.
|
|
193
|
+
if template_height > search_height or template_width > search_width:
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
if float(template_image.std()) < 1e-6:
|
|
197
|
+
# TM_CCOEFF_NORMED is undefined for a constant template.
|
|
198
|
+
raw = cv2.matchTemplate(search, template_image, cv2.TM_SQDIFF_NORMED)
|
|
199
|
+
scores = 1.0 - np.clip(raw, 0.0, 1.0)
|
|
200
|
+
else:
|
|
201
|
+
raw = cv2.matchTemplate(search, template_image, cv2.TM_CCOEFF_NORMED)
|
|
202
|
+
scores = np.clip(raw, -1.0, 1.0)
|
|
203
|
+
|
|
204
|
+
scores = np.nan_to_num(
|
|
205
|
+
scores, nan=-1.0, posinf=1.0, neginf=-1.0
|
|
206
|
+
).astype(np.float32)
|
|
207
|
+
|
|
208
|
+
return scores, offset, (template_width, template_height)
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
def _load_template(template_path: ImageSource) -> np.ndarray:
|
|
212
|
+
if isinstance(template_path, np.ndarray):
|
|
213
|
+
template = template_path
|
|
214
|
+
else:
|
|
215
|
+
path = Path(template_path)
|
|
216
|
+
template = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
|
217
|
+
if template is None:
|
|
218
|
+
raise FileNotFoundError(f"could not read template image: {path}")
|
|
219
|
+
|
|
220
|
+
if not isinstance(template, np.ndarray) or template.size == 0:
|
|
221
|
+
raise ValueError("template must be a non-empty numpy array")
|
|
222
|
+
|
|
223
|
+
return template
|
|
224
|
+
|
|
225
|
+
@staticmethod
|
|
226
|
+
def _prepare_image(image: np.ndarray, grayscale: bool) -> np.ndarray:
|
|
227
|
+
if image.dtype != np.uint8:
|
|
228
|
+
image = np.clip(image, 0, 255).astype(np.uint8)
|
|
229
|
+
|
|
230
|
+
if image.ndim == 2:
|
|
231
|
+
if grayscale:
|
|
232
|
+
return np.ascontiguousarray(image)
|
|
233
|
+
return np.ascontiguousarray(cv2.cvtColor(image, cv2.COLOR_GRAY2BGR))
|
|
234
|
+
|
|
235
|
+
if image.ndim != 3:
|
|
236
|
+
raise ValueError("image must be 2D grayscale or 3D color")
|
|
237
|
+
|
|
238
|
+
channels = image.shape[2]
|
|
239
|
+
if channels == 4:
|
|
240
|
+
image = cv2.cvtColor(image, cv2.COLOR_BGRA2BGR)
|
|
241
|
+
elif channels != 3:
|
|
242
|
+
raise ValueError("image must have 1, 3, or 4 channels")
|
|
243
|
+
|
|
244
|
+
if grayscale:
|
|
245
|
+
return np.ascontiguousarray(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
|
|
246
|
+
|
|
247
|
+
return np.ascontiguousarray(image)
|
|
248
|
+
|
|
249
|
+
@staticmethod
|
|
250
|
+
def _iou(first: MatchResult, second: MatchResult) -> float:
|
|
251
|
+
first_x2 = first.x + first.width
|
|
252
|
+
first_y2 = first.y + first.height
|
|
253
|
+
second_x2 = second.x + second.width
|
|
254
|
+
second_y2 = second.y + second.height
|
|
255
|
+
|
|
256
|
+
intersection_width = max(
|
|
257
|
+
0, min(first_x2, second_x2) - max(first.x, second.x)
|
|
258
|
+
)
|
|
259
|
+
intersection_height = max(
|
|
260
|
+
0, min(first_y2, second_y2) - max(first.y, second.y)
|
|
261
|
+
)
|
|
262
|
+
intersection = intersection_width * intersection_height
|
|
263
|
+
|
|
264
|
+
if intersection == 0:
|
|
265
|
+
return 0.0
|
|
266
|
+
|
|
267
|
+
union = (
|
|
268
|
+
first.width * first.height
|
|
269
|
+
+ second.width * second.height
|
|
270
|
+
- intersection
|
|
271
|
+
)
|
|
272
|
+
return intersection / union if union else 0.0
|
pyvision_bot/models.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Data structures used by :mod:`pyvision_bot`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Tuple, Union
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class ROI:
|
|
11
|
+
"""A rectangular region of interest in screen coordinates.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
x: Left edge of the region, in pixels.
|
|
15
|
+
y: Top edge of the region, in pixels.
|
|
16
|
+
w: Width of the region, in pixels.
|
|
17
|
+
h: Height of the region, in pixels.
|
|
18
|
+
|
|
19
|
+
Raises:
|
|
20
|
+
ValueError: If ``w`` or ``h`` is not positive.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
x: int
|
|
24
|
+
y: int
|
|
25
|
+
w: int
|
|
26
|
+
h: int
|
|
27
|
+
|
|
28
|
+
def __post_init__(self) -> None:
|
|
29
|
+
if self.w <= 0 or self.h <= 0:
|
|
30
|
+
raise ValueError("ROI width and height must be positive")
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def box(self) -> Tuple[int, int, int, int]:
|
|
34
|
+
"""Return the region as an ``(x, y, w, h)`` tuple."""
|
|
35
|
+
|
|
36
|
+
return (self.x, self.y, self.w, self.h)
|
|
37
|
+
|
|
38
|
+
def clamp(self, screen_width: int, screen_height: int) -> "ROI":
|
|
39
|
+
"""Clip the region to the screen bounds.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
screen_width: Width of the screenshot in pixels.
|
|
43
|
+
screen_height: Height of the screenshot in pixels.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
A new :class:`ROI` that is fully inside the screen.
|
|
47
|
+
|
|
48
|
+
Raises:
|
|
49
|
+
ValueError: If the screen size is invalid or the ROI does not
|
|
50
|
+
overlap the screen at all.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
if screen_width <= 0 or screen_height <= 0:
|
|
54
|
+
raise ValueError("screen dimensions must be positive")
|
|
55
|
+
|
|
56
|
+
x1 = max(0, self.x)
|
|
57
|
+
y1 = max(0, self.y)
|
|
58
|
+
x2 = min(screen_width, self.x + self.w)
|
|
59
|
+
y2 = min(screen_height, self.y + self.h)
|
|
60
|
+
|
|
61
|
+
if x2 <= x1 or y2 <= y1:
|
|
62
|
+
raise ValueError("ROI does not overlap the screen")
|
|
63
|
+
|
|
64
|
+
return ROI(x1, y1, x2 - x1, y2 - y1)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
ROIType = Union[ROI, Tuple[int, int, int, int]]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def normalize_roi(roi: ROIType) -> ROI:
|
|
71
|
+
"""Convert an :class:`ROI` or ``(x, y, w, h)`` tuple into an :class:`ROI`.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
roi: Region of interest to normalize.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
The normalized :class:`ROI`.
|
|
78
|
+
|
|
79
|
+
Raises:
|
|
80
|
+
TypeError: If ``roi`` is neither an :class:`ROI` nor a 4-tuple.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
if isinstance(roi, ROI):
|
|
84
|
+
return roi
|
|
85
|
+
|
|
86
|
+
if isinstance(roi, tuple) and len(roi) == 4:
|
|
87
|
+
x, y, w, h = roi
|
|
88
|
+
return ROI(int(x), int(y), int(w), int(h))
|
|
89
|
+
|
|
90
|
+
raise TypeError("roi must be an ROI or a (x, y, w, h) tuple")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass(frozen=True)
|
|
94
|
+
class MatchResult:
|
|
95
|
+
"""A successful template match.
|
|
96
|
+
|
|
97
|
+
Coordinates are absolute screenshot coordinates, even when a ROI was
|
|
98
|
+
used for the search.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
x: Left edge of the match, in pixels.
|
|
102
|
+
y: Top edge of the match, in pixels.
|
|
103
|
+
width: Width of the matched template, in pixels.
|
|
104
|
+
height: Height of the matched template, in pixels.
|
|
105
|
+
confidence: Normalized match confidence in ``[0, 1]``.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
x: int
|
|
109
|
+
y: int
|
|
110
|
+
width: int
|
|
111
|
+
height: int
|
|
112
|
+
confidence: float
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def center(self) -> Tuple[int, int]:
|
|
116
|
+
"""Return the center point of the match as ``(cx, cy)``."""
|
|
117
|
+
|
|
118
|
+
return (self.x + self.width // 2, self.y + self.height // 2)
|
pyvision_bot/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyvision-bot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: OpenCV image automation for desktop and Android (ADB) with template matching.
|
|
5
|
+
Author: 邱祺峰
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Chifeng-chiu/_se/tree/main/HW8/pyvision-bot
|
|
8
|
+
Project-URL: Repository, https://github.com/Chifeng-chiu/_se
|
|
9
|
+
Project-URL: Issues, https://github.com/Chifeng-chiu/_se/issues
|
|
10
|
+
Keywords: automation,opencv,template-matching,adb,android,gui,rpa
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Classifier: Topic :: Software Development :: User Interfaces
|
|
22
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: opencv-python>=4.8.0
|
|
27
|
+
Requires-Dist: numpy>=1.23.0
|
|
28
|
+
Requires-Dist: mss>=9.0.1
|
|
29
|
+
Requires-Dist: pyautogui>=0.9.54
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
32
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
33
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# pyvision-bot
|
|
37
|
+
|
|
38
|
+
[](https://pypi.org/project/pyvision-bot/)
|
|
39
|
+
[](https://pypi.org/project/pyvision-bot/)
|
|
40
|
+
[](LICENSE)
|
|
41
|
+
|
|
42
|
+
以 **OpenCV 模板比對**驅動的影像自動化套件,同時支援 **Windows / macOS / Linux 桌面**與 **Android(ADB)**。用一張按鈕截圖就能等待畫面出現、點擊、拖曳,適合 GUI 測試、RPA 與手機遊戲自動化。
|
|
43
|
+
|
|
44
|
+
## 特色
|
|
45
|
+
|
|
46
|
+
* **雙模式後端**:`DesktopBackend`(`mss` 截圖 + `pyautogui` 操作)與 `ADBBackend`(`adb exec-out screencap` + `adb shell input`)
|
|
47
|
+
* **OpenCV 找圖**:`cv2.matchTemplate` + `TM_CCOEFF_NORMED`,支援灰階加速與 ROI 裁切
|
|
48
|
+
* **座標正確**:使用 ROI 時,回傳的仍是全螢幕絕對座標
|
|
49
|
+
* **多目標偵測**:`find_all()` 內建非極大值抑制(NMS),同一個目標不會重複回報
|
|
50
|
+
* **擬人化操作**:點擊/滑動支援 `jitter` 隨機偏移與隨機微延遲
|
|
51
|
+
* **等待流程**:`wait_for()`、`wait_and_click()` 自動輪詢直到目標出現或逾時
|
|
52
|
+
* **不落地截圖**:ADB 截圖直接在記憶體用 `cv2.imdecode` 解碼,不寫暫存檔
|
|
53
|
+
* **完整型別註記**:所有公開 API 都有 Type Hints 與 Docstrings,附 `py.typed`
|
|
54
|
+
|
|
55
|
+
## 安裝
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install pyvision-bot
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
從原始碼安裝(含開發工具):
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
git clone https://github.com/Chifeng-chiu/_se.git
|
|
65
|
+
cd _se/HW8/pyvision-bot
|
|
66
|
+
python -m pip install -e ".[dev]"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
需求:Python 3.9 以上。
|
|
70
|
+
|
|
71
|
+
> Linux 桌面若缺少 `pyautogui` 需要的系統套件,請先安裝 `python3-xlib`、`scrot` 等相依套件;ADB 模式不需要 `pyautogui`。
|
|
72
|
+
|
|
73
|
+
## 快速上手:Desktop
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from pyvision_bot import ROI, VisionBot
|
|
77
|
+
|
|
78
|
+
bot = VisionBot(mode="desktop")
|
|
79
|
+
|
|
80
|
+
# 等待按鈕出現(最多 15 秒),找到後點擊中心,加入 ±3 px 隨機偏移
|
|
81
|
+
clicked = bot.wait_and_click("assets/login_button.png", timeout=15, jitter=3)
|
|
82
|
+
print("clicked:", clicked)
|
|
83
|
+
|
|
84
|
+
# 只在畫面左上角區域找圖
|
|
85
|
+
match = bot.find("assets/icon.png", roi=ROI(0, 0, 1280, 720))
|
|
86
|
+
if match:
|
|
87
|
+
print("center:", match.center, "confidence:", match.confidence)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
* `DesktopBackend` 預設擷取 **主螢幕**(`monitor=1`)。若要擷取所有螢幕的聯集,可建立 `DesktopBackend(monitor=0)` 後注入 `VisionBot(backend=...)`。
|
|
91
|
+
* 座標以擷取到的螢幕左上角為原點;多螢幕且非主螢幕時請自行換算全域座標。
|
|
92
|
+
|
|
93
|
+
## 快速上手:Android(ADB)
|
|
94
|
+
|
|
95
|
+
前置:安裝 Android platform-tools、開啟手機 USB 偵錯,並確認 `adb devices` 看得到裝置。
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
adb devices
|
|
99
|
+
# List of devices attached
|
|
100
|
+
# emulator-5554 device
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from pyvision_bot import VisionBot
|
|
105
|
+
|
|
106
|
+
bot = VisionBot(mode="adb", device_serial="emulator-5554")
|
|
107
|
+
|
|
108
|
+
# 等待遊戲開始按鈕並點擊
|
|
109
|
+
bot.wait_and_click("assets/start.png", timeout=20, jitter=2)
|
|
110
|
+
|
|
111
|
+
# 由下往上滑動
|
|
112
|
+
bot.swipe(540, 1600, 540, 400, duration=0.4, jitter=5)
|
|
113
|
+
|
|
114
|
+
# 取得目前畫面(BGR ndarray)
|
|
115
|
+
screen = bot.screenshot()
|
|
116
|
+
print(screen.shape)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
不指定 `device_serial` 時,ADB 會使用目前唯一的裝置:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
bot = VisionBot(mode="adb")
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## API 說明
|
|
126
|
+
|
|
127
|
+
### `VisionBot`
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
VisionBot(
|
|
131
|
+
mode: Literal["desktop", "adb"] = "desktop",
|
|
132
|
+
device_serial: str | None = None,
|
|
133
|
+
default_threshold: float = 0.85,
|
|
134
|
+
*,
|
|
135
|
+
backend: BaseBackend | None = None,
|
|
136
|
+
matcher: TemplateMatcher | None = None,
|
|
137
|
+
rng: random.Random | None = None,
|
|
138
|
+
)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
| 方法 | 說明 |
|
|
142
|
+
| --- | --- |
|
|
143
|
+
| `screenshot() -> np.ndarray` | 取得目前畫面(BGR) |
|
|
144
|
+
| `find(template_path, threshold=None, roi=None) -> MatchResult \| None` | 找最佳匹配 |
|
|
145
|
+
| `find_all(template_path, threshold=None, roi=None, max_results=None) -> list[MatchResult]` | 找所有不重疊匹配(NMS) |
|
|
146
|
+
| `click(x, y, jitter=0)` | 點擊,支援 ±jitter 像素隨機偏移 |
|
|
147
|
+
| `swipe(x1, y1, x2, y2, duration=0.3, jitter=0)` | 拖曳/滑動 |
|
|
148
|
+
| `wait_for(template_path, timeout=10.0, interval=0.5, threshold=None, roi=None)` | 輪詢等待目標出現 |
|
|
149
|
+
| `wait_and_click(template_path, timeout=10.0, interval=0.5, threshold=None, roi=None, jitter=3)` | 等待目標並點擊中心 |
|
|
150
|
+
|
|
151
|
+
`backend`、`matcher`、`rng` 是進階注入點:測試時可傳入自訂 backend,或傳入固定種子的 `random.Random` 讓 jitter 可重現。
|
|
152
|
+
|
|
153
|
+
### `TemplateMatcher`
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from pyvision_bot import TemplateMatcher
|
|
157
|
+
|
|
158
|
+
matcher = TemplateMatcher(threshold=0.9, grayscale=True, nms_iou=0.3)
|
|
159
|
+
screen = bot.screenshot()
|
|
160
|
+
|
|
161
|
+
best = matcher.find(screen, "assets/target.png")
|
|
162
|
+
all_matches = matcher.find_all(screen, "assets/target.png", max_results=5)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
| 參數 | 預設 | 說明 |
|
|
166
|
+
| --- | --- | --- |
|
|
167
|
+
| `threshold` | `0.85` | 相似度門檻(0~1) |
|
|
168
|
+
| `grayscale` | `True` | 轉灰階比對,速度較快 |
|
|
169
|
+
| `nms_iou` | `0.3` | `find_all()` 的 NMS IoU 門檻 |
|
|
170
|
+
|
|
171
|
+
`template_path` 可以是檔案路徑、`pathlib.Path`,或直接傳入 `numpy` 陣列。
|
|
172
|
+
|
|
173
|
+
### 資料結構
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from pyvision_bot import MatchResult, ROI
|
|
177
|
+
|
|
178
|
+
match = MatchResult(x=100, y=50, width=40, height=30, confidence=0.97)
|
|
179
|
+
print(match.center) # (120, 65)
|
|
180
|
+
|
|
181
|
+
roi = ROI(0, 0, 1280, 720)
|
|
182
|
+
print(roi.box) # (0, 0, 1280, 720)
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
* `MatchResult.center` 回傳 `(cx, cy)`。
|
|
186
|
+
* `ROI` 也接受 `(x, y, w, h)` tuple;超出螢幕的區域會自動裁切,完全在畫面外則拋出 `ValueError`。
|
|
187
|
+
|
|
188
|
+
### 自訂 Backend
|
|
189
|
+
|
|
190
|
+
繼承 `BaseBackend` 即可接入其他來源(例如模擬器、遠端桌面):
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from pyvision_bot import BaseBackend
|
|
194
|
+
import numpy as np
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class MyBackend(BaseBackend):
|
|
198
|
+
def screenshot(self) -> np.ndarray:
|
|
199
|
+
...
|
|
200
|
+
|
|
201
|
+
def click(self, x: int, y: int) -> None:
|
|
202
|
+
...
|
|
203
|
+
|
|
204
|
+
def swipe(self, x1: int, y1: int, x2: int, y2: int, duration: float) -> None:
|
|
205
|
+
...
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## 擬人化行為
|
|
209
|
+
|
|
210
|
+
`click()` 與 `swipe()` 會:
|
|
211
|
+
|
|
212
|
+
1. 在動作前插入 20~80 ms 的隨機延遲
|
|
213
|
+
2. 若 `jitter > 0`,在每個座標軸加入 `±jitter` 像素的隨機偏移
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
bot.click(500, 300, jitter=4)
|
|
217
|
+
bot.swipe(200, 800, 200, 200, duration=0.5, jitter=3)
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
這可以降低固定座標、固定節奏造成的機械感,但**不保證**能繞過任何特定服務的偵測機制。
|
|
221
|
+
|
|
222
|
+
## 開發與測試
|
|
223
|
+
|
|
224
|
+
```bash
|
|
225
|
+
python -m pip install -e ".[dev]"
|
|
226
|
+
pytest
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
測試完全使用合成的 `numpy` 圖片與 Mock Backend,不需要真實螢幕、滑鼠或 ADB 裝置。
|
|
230
|
+
|
|
231
|
+
## 打包與發佈
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
python -m build
|
|
235
|
+
twine check dist/*
|
|
236
|
+
twine upload dist/*
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
發佈前請確認 `pyproject.toml` 的 `authors`、`project.urls` 是你要的資訊,並設定 PyPI API token:
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
# PowerShell
|
|
243
|
+
$env:TWINE_USERNAME = "__token__"
|
|
244
|
+
$env:TWINE_PASSWORD = "pypi-..."
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## 注意事項
|
|
248
|
+
|
|
249
|
+
* 桌面模式的座標與螢幕縮放(DPI scaling)有關;若 Windows 設定 125% / 150% 縮放,請確認截圖與 `pyautogui` 使用相同的座標系。
|
|
250
|
+
* `pyautogui` 預設啟用滑鼠移到左上角即中止的 failsafe,緊急時可把滑鼠移到 (0, 0)。
|
|
251
|
+
* ADB 模式需要裝置已授權此電腦;首次連線請在手機上按「允許 USB 偵錯」。
|
|
252
|
+
* 本套件只提供影像比對與輸入模擬,請僅用於你擁有或獲授權操作的裝置與服務。
|
|
253
|
+
|
|
254
|
+
## License
|
|
255
|
+
|
|
256
|
+
MIT License,詳見 [LICENSE](LICENSE)。
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
pyvision_bot/__init__.py,sha256=jFevJh5yrfRCXhhOPkqKKtjnvsjLhbByBJ0CfnB-gI0,425
|
|
2
|
+
pyvision_bot/backends.py,sha256=So60RNHZ_0po2_7-dPaW68JDzcenBUtu_EGpVc2v1wA,7083
|
|
3
|
+
pyvision_bot/bot.py,sha256=pXlyYi28VD7I-61zaNSuK7feyR4CElT0U89KBVbvCeg,9226
|
|
4
|
+
pyvision_bot/matcher.py,sha256=oP-AmPtAZWSsu7gRuDwBQ9Za6jKWKwvcbSduKKGM0CQ,9841
|
|
5
|
+
pyvision_bot/models.py,sha256=baB12jSAR5y4_JLSD28vCNrI2NDS6uIi3Jj76sT3FK8,3144
|
|
6
|
+
pyvision_bot/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
pyvision_bot-0.1.0.dist-info/licenses/LICENSE,sha256=iH6XNYS-flpSYO8JRw_0VtFa95XW3YSvYXjEXKRXU1k,1066
|
|
8
|
+
pyvision_bot-0.1.0.dist-info/METADATA,sha256=Bm47H7K0Ta2zZMQ7DqO6GtlJnUdf1-r_1ufr2eLRQQc,9002
|
|
9
|
+
pyvision_bot-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
pyvision_bot-0.1.0.dist-info/top_level.txt,sha256=WcR3lhSRxitdTa7oqkmpahz2YijU2Ip6F0g0DkQvimo,13
|
|
11
|
+
pyvision_bot-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 邱祺峰
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pyvision_bot
|