dynamic-test-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- automation/__init__.py +0 -0
- automation/config.py +69 -0
- automation/engines/__init__.py +0 -0
- automation/engines/engine_manager.py +382 -0
- automation/engines/uia_engine.py +220 -0
- automation/engines/vision_base.py +30 -0
- automation/engines/vision_engine.py +474 -0
- automation/engines/windows_uia_engine.py +347 -0
- automation/keywords/__init__.py +0 -0
- automation/keywords/automation_keywords.py +83 -0
- automation/utils/__init__.py +0 -0
- automation/utils/adb.py +220 -0
- automation/utils/cache.py +135 -0
- automation/utils/device_interface.py +60 -0
- automation/utils/image_processing.py +258 -0
- automation/utils/scroll_centering.py +48 -0
- automation/utils/spatial.py +63 -0
- automation/utils/windows_host.py +354 -0
- dynamic_test_engine-0.1.0.dist-info/METADATA +405 -0
- dynamic_test_engine-0.1.0.dist-info/RECORD +23 -0
- dynamic_test_engine-0.1.0.dist-info/WHEEL +5 -0
- dynamic_test_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
- dynamic_test_engine-0.1.0.dist-info/top_level.txt +1 -0
automation/__init__.py
ADDED
|
File without changes
|
automation/config.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import logging
|
|
3
|
+
from dotenv import load_dotenv
|
|
4
|
+
|
|
5
|
+
load_dotenv()
|
|
6
|
+
|
|
7
|
+
# Logging-Konfiguration: steuert Sichtbarkeit aller automation.*-Logger.
|
|
8
|
+
# Werte: DEBUG | INFO | WARNING | ERROR. Details: docs/konfiguration.md
|
|
9
|
+
LOG_LEVEL = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
|
|
10
|
+
|
|
11
|
+
# Nur den eigenen Logger konfigurieren, nicht den Root-Logger des Host-Prozesses
|
|
12
|
+
# (robot, pytest, ...) - eine Library soll dessen Logging-Setup nicht überschreiben.
|
|
13
|
+
# Hat der Host bereits einen Root-Handler, übernimmt dessen Ausgabe per Propagation
|
|
14
|
+
# (kein eigener Handler, sonst doppelte Zeilen). Sonst eigener Handler als Fallback,
|
|
15
|
+
# damit Logs auch bei direktem Skriptaufruf ohne Host-Logging sichtbar sind.
|
|
16
|
+
_automation_logger = logging.getLogger("automation")
|
|
17
|
+
if not logging.getLogger().handlers and not _automation_logger.handlers:
|
|
18
|
+
_handler = logging.StreamHandler()
|
|
19
|
+
_handler.setFormatter(logging.Formatter(
|
|
20
|
+
"%(asctime)s %(levelname)s [%(name)s] %(message)s",
|
|
21
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
22
|
+
))
|
|
23
|
+
_automation_logger.addHandler(_handler)
|
|
24
|
+
_automation_logger.setLevel(LOG_LEVEL)
|
|
25
|
+
|
|
26
|
+
DEVICE_ID = os.getenv("ANDROID_DEVICE_ID", "")
|
|
27
|
+
DEFAULT_ENGINE = os.getenv("DEFAULT_ENGINE", "auto")
|
|
28
|
+
MATCH_THRESHOLD = float(os.getenv("CUSTOM_ELEMENT_MATCH_THRESHOLD", "0.9"))
|
|
29
|
+
OCR_LANG = os.getenv("OCR_LANGUAGES", "deu+eng")
|
|
30
|
+
OCR_PSM = int(os.getenv("OCR_PSM", "11"))
|
|
31
|
+
OCR_OEM = int(os.getenv("OCR_OEM", "1"))
|
|
32
|
+
OCR_SCALE = float(os.getenv("OCR_SCALE", "1.0"))
|
|
33
|
+
SCROLL_RETRIES = int(os.getenv("SCROLL_RETRIES", "3"))
|
|
34
|
+
VISION_SCROLL_MAX = int(os.getenv("VISION_SCROLL_MAX", "30"))
|
|
35
|
+
# Details zu den Scroll-Werten: docs/konfiguration.md
|
|
36
|
+
VISION_SCROLL_STEP_PERCENT = float(os.getenv("VISION_SCROLL_STEP_PERCENT", "0.20"))
|
|
37
|
+
SCROLL_SAFE_MARGIN_PERCENT = float(os.getenv("SCROLL_SAFE_MARGIN_PERCENT", "0.25"))
|
|
38
|
+
ADB_TIMEOUT = int(os.getenv("ADB_TIMEOUT", "3"))
|
|
39
|
+
ADB_SCREENSHOT_TIMEOUT = int(os.getenv("ADB_SCREENSHOT_TIMEOUT", "10"))
|
|
40
|
+
ADB_SWIPE_DURATION_MS = int(os.getenv("ADB_SWIPE_DURATION_MS", "300"))
|
|
41
|
+
CENTERING_SWIPE_DURATION_MS = int(os.getenv("CENTERING_SWIPE_DURATION_MS", "500"))
|
|
42
|
+
VISION_SCROLL_SWIPE_DURATION_MS = int(os.getenv("VISION_SCROLL_SWIPE_DURATION_MS", "200"))
|
|
43
|
+
OCR_CONFIDENCE = int(os.getenv("OCR_CONFIDENCE", "60"))
|
|
44
|
+
UIA_TIMEOUT = int(os.getenv("UIA_TIMEOUT", "3"))
|
|
45
|
+
UIA_SCROLL_TIMEOUT = int(os.getenv("UIA_SCROLL_TIMEOUT", "8"))
|
|
46
|
+
# Maximale Anzahl diskreter Swipes in der Lernschleife (scroll_to_text).
|
|
47
|
+
UIA_SCROLL_MAX_SWIPES = int(os.getenv("UIA_SCROLL_MAX_SWIPES", "30"))
|
|
48
|
+
UIA_WAIT_IDLE_TIMEOUT = float(os.getenv("UIA_WAIT_IDLE_TIMEOUT", "0.5"))
|
|
49
|
+
UIA_APP_START_TIMEOUT = int(os.getenv("UIA_APP_START_TIMEOUT", "3"))
|
|
50
|
+
# Details zu den Mausrad-Werten (auflösungsabhängig, selbst zu justieren): docs/konfiguration.md
|
|
51
|
+
WINDOWS_SCROLL_NOTCHES = int(os.getenv("WINDOWS_SCROLL_NOTCHES", "10"))
|
|
52
|
+
WINDOWS_SCROLL_EVENT_DELAY = float(os.getenv("WINDOWS_SCROLL_EVENT_DELAY", "0.15"))
|
|
53
|
+
WINDOWS_CLEAR_FIELD_DELAY = float(os.getenv("WINDOWS_CLEAR_FIELD_DELAY", "0.1"))
|
|
54
|
+
SCREENSHOT_DIR = os.getenv("SCREENSHOT_DIR", "logs/")
|
|
55
|
+
CUSTOM_ELEMENTS_DIR = os.getenv("CUSTOM_ELEMENTS_DIR", "custom_elements/")
|
|
56
|
+
APK_PATH = os.getenv("APK_PATH", "")
|
|
57
|
+
JPG_QUALITY = int(os.getenv("SCREENSHOT_JPG_QUALITY", "85"))
|
|
58
|
+
VISION_STABLE_THRESHOLD = float(os.getenv("VISION_STABLE_THRESHOLD", "2.0"))
|
|
59
|
+
|
|
60
|
+
# Caching (Details: docs/konfiguration.md, docs/recherche_caching.md)
|
|
61
|
+
UIA_DUMP_CACHE = os.getenv("UIA_DUMP_CACHE", "false").lower() == "true"
|
|
62
|
+
UIA_CACHE_DIR = os.getenv("UIA_CACHE_DIR", "logs/cache/")
|
|
63
|
+
VISION_OCR_CACHE = os.getenv("VISION_OCR_CACHE", "false").lower() == "true"
|
|
64
|
+
VISION_CACHE_DIR = os.getenv("VISION_CACHE_DIR", "logs/cache/vision/")
|
|
65
|
+
VISION_CROP_PAD = int(os.getenv("VISION_CROP_PAD", "25"))
|
|
66
|
+
|
|
67
|
+
# Räumliche Relationen (Find/Tap Element), Details: docs/konfiguration.md
|
|
68
|
+
SPATIAL_OVERLAP_TOLERANCE = int(os.getenv("SPATIAL_OVERLAP_TOLERANCE", "10"))
|
|
69
|
+
SPATIAL_NEAR_MAX_DISTANCE = int(os.getenv("SPATIAL_NEAR_MAX_DISTANCE", "0"))
|
|
File without changes
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from http.client import RemoteDisconnected
|
|
8
|
+
from typing import Any, Callable, TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from automation import config
|
|
14
|
+
from automation.engines.uia_engine import UiaEngine
|
|
15
|
+
from automation.engines.vision_base import VisionEngineBase
|
|
16
|
+
from automation.engines.vision_engine import VisionEngine
|
|
17
|
+
from automation.utils import adb
|
|
18
|
+
from automation.utils.device_interface import DeviceInterface
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
_uia = UiaEngine()
|
|
23
|
+
_device: DeviceInterface = adb.AdbDevice()
|
|
24
|
+
_vision: VisionEngineBase = VisionEngine(_device)
|
|
25
|
+
_platform: str = "android"
|
|
26
|
+
|
|
27
|
+
def _uia_relation_not_impl() -> Any:
|
|
28
|
+
raise NotImplementedError(
|
|
29
|
+
"Find/Tap Element ist für UIA noch nicht implementiert (DOM-Resolver folgt). "
|
|
30
|
+
"engine='vision' verwenden."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class EngineManager:
|
|
35
|
+
|
|
36
|
+
def _run_explicit(self, action: str, fn: Callable[[], Any], log_label: str, error_label: str) -> Any:
|
|
37
|
+
logger.info("%s: engine=%s", action, log_label)
|
|
38
|
+
try:
|
|
39
|
+
return fn()
|
|
40
|
+
except Exception as e:
|
|
41
|
+
path = self._save_error_screenshot(e)
|
|
42
|
+
raise RuntimeError(f"{action} failed ({error_label}): {e}. Screenshot: {path}") from e
|
|
43
|
+
|
|
44
|
+
def _run(self, action: str, uia_fn: Callable[[], Any], vision_fn: Callable[[], Any], engine: str = "auto") -> Any:
|
|
45
|
+
if engine == "uia":
|
|
46
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
47
|
+
if engine == "vision":
|
|
48
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
49
|
+
# auto
|
|
50
|
+
try:
|
|
51
|
+
result = uia_fn()
|
|
52
|
+
logger.info("%s: engine=uia", action)
|
|
53
|
+
return result
|
|
54
|
+
except Exception as uia_error:
|
|
55
|
+
logger.warning("%s: uia failed (%s), fallback to vision", action, uia_error)
|
|
56
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
57
|
+
|
|
58
|
+
def _uia_bool_attempt(self, action: str, uia_fn: Callable[[], Any]) -> bool:
|
|
59
|
+
try:
|
|
60
|
+
found = uia_fn()
|
|
61
|
+
except Exception as uia_error:
|
|
62
|
+
logger.warning("%s: uia failed (%s), fallback to vision", action, uia_error)
|
|
63
|
+
return False
|
|
64
|
+
logger.info("%s: engine=uia", action)
|
|
65
|
+
return found
|
|
66
|
+
|
|
67
|
+
def _run_bool(self, action: str, uia_fn: Callable[[], Any], vision_fn: Callable[[], Any], engine: str = "auto") -> bool:
|
|
68
|
+
if engine == "uia":
|
|
69
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
70
|
+
if engine == "vision":
|
|
71
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
72
|
+
if self._uia_bool_attempt(action, uia_fn):
|
|
73
|
+
return True
|
|
74
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
75
|
+
|
|
76
|
+
def _write_screenshot(self, img: np.ndarray, prefix: str) -> str:
|
|
77
|
+
import cv2
|
|
78
|
+
os.makedirs(config.SCREENSHOT_DIR, exist_ok=True)
|
|
79
|
+
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
80
|
+
path = os.path.join(config.SCREENSHOT_DIR, f"{prefix}_{ts}.jpg")
|
|
81
|
+
if not cv2.imwrite(path, img, [cv2.IMWRITE_JPEG_QUALITY, config.JPG_QUALITY]):
|
|
82
|
+
raise RuntimeError(f"Screenshot write failed: {path}")
|
|
83
|
+
return path
|
|
84
|
+
|
|
85
|
+
def _save_error_screenshot(self, error: Exception | None = None) -> str:
|
|
86
|
+
try:
|
|
87
|
+
img = getattr(error, "screenshot", None)
|
|
88
|
+
if img is None:
|
|
89
|
+
img = _device.screenshot()
|
|
90
|
+
path = self._write_screenshot(img, "error")
|
|
91
|
+
return path
|
|
92
|
+
except Exception as e:
|
|
93
|
+
logger.debug("Error screenshot failed: %s", e, exc_info=True)
|
|
94
|
+
os.makedirs(config.SCREENSHOT_DIR, exist_ok=True)
|
|
95
|
+
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
96
|
+
return os.path.join(config.SCREENSHOT_DIR, f"error_{ts}.jpg (screenshot failed)")
|
|
97
|
+
|
|
98
|
+
def connect_device(self, device_id: str | None = None, platform: str = "android") -> None:
|
|
99
|
+
global _uia, _device, _vision, _platform
|
|
100
|
+
_platform = platform
|
|
101
|
+
if platform == "windows":
|
|
102
|
+
from automation.engines.windows_uia_engine import WindowsUiaEngine # lazy: Windows-only
|
|
103
|
+
from automation.utils.windows_host import WindowsHost # lazy: Windows-only
|
|
104
|
+
_device = WindowsHost()
|
|
105
|
+
_uia = WindowsUiaEngine(_device)
|
|
106
|
+
try:
|
|
107
|
+
_device.connect(device_id)
|
|
108
|
+
except RuntimeError as e:
|
|
109
|
+
# Verbindung fehlgeschlagen (z. B. Prozess läuft noch nicht). Kein Fehler an
|
|
110
|
+
# sich, wenn direkt danach Launch App aufgerufen wird, das startet den Prozess
|
|
111
|
+
# frisch und verbindet neu. UIA bleibt bis dahin nachweislich getrennt
|
|
112
|
+
# (WindowsHost.app is None), nachfolgende Aufrufe fallen im auto-Modus auf
|
|
113
|
+
# Vision zurück statt unauffällig einen falschen Verbindungsstatus vorzutäuschen.
|
|
114
|
+
logger.warning(
|
|
115
|
+
"connect_device: Windows-Verbindung zu '%s' fehlgeschlagen (%s). "
|
|
116
|
+
"UIA bleibt getrennt bis Launch App neu verbindet.", device_id, e
|
|
117
|
+
)
|
|
118
|
+
_vision = VisionEngine(_device)
|
|
119
|
+
else:
|
|
120
|
+
effective_id = device_id or config.DEVICE_ID or None
|
|
121
|
+
_uia.connect(effective_id)
|
|
122
|
+
_device.connect(effective_id)
|
|
123
|
+
|
|
124
|
+
def install_app(self, apk_path: str) -> None:
|
|
125
|
+
if _platform == "windows":
|
|
126
|
+
raise NotImplementedError(
|
|
127
|
+
"Install App ist auf Windows nicht verfügbar. App manuell starten."
|
|
128
|
+
)
|
|
129
|
+
adb.install_app(apk_path)
|
|
130
|
+
|
|
131
|
+
def launch_app(self, package: str, activity: str | None = None, fullscreen: bool = False) -> None:
|
|
132
|
+
if _platform == "windows":
|
|
133
|
+
# _uia.launch_app aktualisiert die zentrale Host-Verbindung selbst
|
|
134
|
+
# (WindowsHost.connect mit stabilem process_name), kein zweiter Connect hier nötig.
|
|
135
|
+
_uia.launch_app(package, activity=activity, fullscreen=fullscreen)
|
|
136
|
+
return
|
|
137
|
+
try:
|
|
138
|
+
_uia.launch_app(package, activity=activity)
|
|
139
|
+
except RemoteDisconnected:
|
|
140
|
+
logger.warning("launch_app: RemoteDisconnected, retry nach 1s")
|
|
141
|
+
time.sleep(1)
|
|
142
|
+
_uia.launch_app(package, activity=activity)
|
|
143
|
+
|
|
144
|
+
def close_app(self) -> None:
|
|
145
|
+
if _platform == "windows":
|
|
146
|
+
_uia.close_app()
|
|
147
|
+
return
|
|
148
|
+
raise NotImplementedError(
|
|
149
|
+
"Close App ist auf Android nicht verfügbar. Launch App stoppt die App bereits vor dem Neustart."
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def tap_text(self, text: str, engine: str = "auto") -> None:
|
|
153
|
+
if _platform == "windows":
|
|
154
|
+
return self._tap_text_windows(text, engine)
|
|
155
|
+
self._run(
|
|
156
|
+
f"tap_text('{text}')",
|
|
157
|
+
lambda: _uia.tap_text(text),
|
|
158
|
+
lambda: _vision.tap_text(text),
|
|
159
|
+
engine,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
def _tap_text_windows(self, text: str, engine: str) -> None:
|
|
163
|
+
"""Windows-Sonderpfad für Tap Text: im auto-Modus fällt nur ein echter Miss
|
|
164
|
+
(TextNotFoundError) auf Vision zurück, technische UIA-Fehler werden sichtbar."""
|
|
165
|
+
from automation.engines.windows_uia_engine import TextNotFoundError
|
|
166
|
+
|
|
167
|
+
action = f"tap_text('{text}')"
|
|
168
|
+
|
|
169
|
+
def uia_fn():
|
|
170
|
+
return _uia.tap_text(text)
|
|
171
|
+
|
|
172
|
+
def vision_fn():
|
|
173
|
+
return _vision.tap_text(text)
|
|
174
|
+
|
|
175
|
+
if engine == "uia":
|
|
176
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
177
|
+
if engine == "vision":
|
|
178
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
179
|
+
|
|
180
|
+
try:
|
|
181
|
+
uia_fn()
|
|
182
|
+
logger.info("%s: engine=uia", action)
|
|
183
|
+
return
|
|
184
|
+
except TextNotFoundError as e:
|
|
185
|
+
logger.info("%s: uia miss (%s), fallback to vision", action, e)
|
|
186
|
+
|
|
187
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
188
|
+
|
|
189
|
+
def tap_icon_near_text(self, icon_path: str, text: str, engine: str = "auto") -> None:
|
|
190
|
+
if _platform == "windows":
|
|
191
|
+
return self._tap_icon_near_text_windows(icon_path, text, engine)
|
|
192
|
+
self._run(
|
|
193
|
+
f"tap_icon_near_text('{text}')",
|
|
194
|
+
lambda: _uia.tap_icon_near_text(icon_path, text),
|
|
195
|
+
lambda: _vision.tap_icon_near_text(icon_path, text),
|
|
196
|
+
engine,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def _tap_icon_near_text_windows(self, icon_path: str, text: str, engine: str) -> None:
|
|
200
|
+
"""Windows-Sonderpfad für Tap Icon Near Text: im auto-Modus fällt nur ein echter Miss
|
|
201
|
+
(IconNotFoundError) auf Vision zurück, technische UIA-Fehler werden sichtbar."""
|
|
202
|
+
from automation.engines.windows_uia_engine import IconNotFoundError
|
|
203
|
+
|
|
204
|
+
action = f"tap_icon_near_text('{text}')"
|
|
205
|
+
|
|
206
|
+
def uia_fn():
|
|
207
|
+
return _uia.tap_icon_near_text(icon_path, text)
|
|
208
|
+
|
|
209
|
+
def vision_fn():
|
|
210
|
+
return _vision.tap_icon_near_text(icon_path, text)
|
|
211
|
+
|
|
212
|
+
if engine == "uia":
|
|
213
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
214
|
+
if engine == "vision":
|
|
215
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
uia_fn()
|
|
219
|
+
logger.info("%s: engine=uia", action)
|
|
220
|
+
return
|
|
221
|
+
except IconNotFoundError as e:
|
|
222
|
+
logger.info("%s: uia miss (%s), fallback to vision", action, e)
|
|
223
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
224
|
+
except Exception as e:
|
|
225
|
+
logger.warning("%s: uia technical error (%s), no fallback", action, e)
|
|
226
|
+
path = self._save_error_screenshot(e)
|
|
227
|
+
raise RuntimeError(f"{action} failed (uia): {e}. Screenshot: {path}") from e
|
|
228
|
+
|
|
229
|
+
def input_text(self, locator: str, text: str, anchor: str | None = None,
|
|
230
|
+
engine: str = "auto") -> None:
|
|
231
|
+
if _platform == "windows":
|
|
232
|
+
return self._input_text_windows(locator, text, anchor, engine)
|
|
233
|
+
self._run(
|
|
234
|
+
f"input_text('{locator}')",
|
|
235
|
+
lambda: _uia.input_text(locator, text, anchor=anchor),
|
|
236
|
+
lambda: _vision.input_text(locator, text),
|
|
237
|
+
engine,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
def _input_text_windows(self, locator: str, text: str, anchor: str | None, engine: str) -> None:
|
|
241
|
+
"""Windows-Sonderpfad für Input Text: im auto-Modus fällt nur ein echter Miss
|
|
242
|
+
(FieldNotFoundError) auf Vision zurück, technische Fehler und Mehrdeutigkeit
|
|
243
|
+
werden sichtbar statt roh durchgereicht."""
|
|
244
|
+
from automation.engines.windows_uia_engine import FieldNotFoundError
|
|
245
|
+
|
|
246
|
+
action = f"input_text('{locator}')"
|
|
247
|
+
|
|
248
|
+
def uia_fn():
|
|
249
|
+
return _uia.input_text(locator, text, anchor=anchor)
|
|
250
|
+
|
|
251
|
+
def vision_fn():
|
|
252
|
+
return _vision.input_text(locator, text)
|
|
253
|
+
|
|
254
|
+
if engine == "uia":
|
|
255
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
256
|
+
if engine == "vision":
|
|
257
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
258
|
+
|
|
259
|
+
try:
|
|
260
|
+
uia_fn()
|
|
261
|
+
logger.info("%s: engine=uia", action)
|
|
262
|
+
return
|
|
263
|
+
except FieldNotFoundError as e:
|
|
264
|
+
logger.info("%s: uia miss (%s), fallback to vision", action, e)
|
|
265
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
266
|
+
except Exception as e:
|
|
267
|
+
logger.warning("%s: uia technical error (%s), no fallback", action, e)
|
|
268
|
+
path = self._save_error_screenshot(e)
|
|
269
|
+
raise RuntimeError(f"{action} failed (uia): {e}. Screenshot: {path}") from e
|
|
270
|
+
|
|
271
|
+
def field_value_should_be(self, locator: str, expected: str, anchor: str | None = None,
|
|
272
|
+
engine: str = "uia") -> None:
|
|
273
|
+
"""Windows-only, kein Vision-Pfad, kein auto, kein Fallback: prüft den tatsächlichen
|
|
274
|
+
UIA-Feldwert (ValuePattern) gegen expected."""
|
|
275
|
+
if _platform != "windows":
|
|
276
|
+
raise NotImplementedError("Field Value Should Be ist nur auf Windows verfügbar.")
|
|
277
|
+
if engine != "uia":
|
|
278
|
+
raise ValueError(f"Field Value Should Be unterstützt nur engine='uia', nicht '{engine}'.")
|
|
279
|
+
action = f"field_value_should_be('{locator}')"
|
|
280
|
+
self._run_explicit(
|
|
281
|
+
action, lambda: _uia.field_value_should_be(locator, expected, anchor=anchor), "uia", "uia"
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
def scroll_to_text(self, text: str, anchor: str | None = None, relation: str = "near",
|
|
285
|
+
engine: str = "auto") -> bool:
|
|
286
|
+
return self._run_bool(
|
|
287
|
+
f"scroll_to_text('{text}')",
|
|
288
|
+
lambda: _uia.scroll_to_text(text),
|
|
289
|
+
lambda: _vision.scroll_to_text(text, anchor=anchor, relation=relation),
|
|
290
|
+
engine,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
def text_exists(self, text: str, engine: str = "auto") -> bool:
|
|
294
|
+
if _platform == "windows":
|
|
295
|
+
return self._text_exists_windows(text, engine)
|
|
296
|
+
return self._run_bool(
|
|
297
|
+
f"text_exists('{text}')",
|
|
298
|
+
lambda: _uia.text_exists(text),
|
|
299
|
+
lambda: _vision.text_exists(text),
|
|
300
|
+
engine,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
def _text_exists_windows(self, text: str, engine: str) -> bool:
|
|
304
|
+
"""Windows-Sonderpfad für Text Should Exist: im auto-Modus fällt nur ein echter
|
|
305
|
+
UIA-Miss (False) auf Vision zurück, technische UIA-Fehler werden sichtbar."""
|
|
306
|
+
action = f"text_exists('{text}')"
|
|
307
|
+
|
|
308
|
+
def uia_fn():
|
|
309
|
+
return _uia.text_exists(text)
|
|
310
|
+
|
|
311
|
+
def vision_fn():
|
|
312
|
+
return _vision.text_exists(text)
|
|
313
|
+
if engine == "uia":
|
|
314
|
+
return self._run_explicit(action, uia_fn, "uia", "uia")
|
|
315
|
+
if engine == "vision":
|
|
316
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
317
|
+
found = self._run_explicit(action, uia_fn, "uia", "uia")
|
|
318
|
+
if found:
|
|
319
|
+
return True
|
|
320
|
+
logger.info("%s: uia miss, fallback to vision", action)
|
|
321
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
322
|
+
|
|
323
|
+
def take_screenshot(self, name: str = "screenshot") -> str:
|
|
324
|
+
img = _device.screenshot()
|
|
325
|
+
return self._write_screenshot(img, name)
|
|
326
|
+
|
|
327
|
+
def move_mouse(self, x: int, y: int) -> None:
|
|
328
|
+
_device.move_mouse(x, y)
|
|
329
|
+
|
|
330
|
+
def find_element(self, target: str, anchor: str, relation: str = "near",
|
|
331
|
+
kind: str = "auto", engine: str = "auto") -> tuple[int, int]:
|
|
332
|
+
if _platform == "windows":
|
|
333
|
+
return self._find_element_windows(target, anchor, relation, kind, engine)
|
|
334
|
+
return self._run(
|
|
335
|
+
f"find_element('{target}' {relation} '{anchor}')",
|
|
336
|
+
_uia_relation_not_impl,
|
|
337
|
+
lambda: _vision.find_element(target, anchor, relation=relation, kind=kind),
|
|
338
|
+
engine,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
def _find_element_windows(self, target: str, anchor: str, relation: str, kind: str,
|
|
342
|
+
engine: str) -> tuple[int, int]:
|
|
343
|
+
"""Windows-Dispatch für Find Element: UIA hat noch keinen echten Pfad, loggt das
|
|
344
|
+
deshalb als 'nicht implementiert' statt fälschlich als Fallback-Fehlschlag."""
|
|
345
|
+
action = f"find_element('{target}' {relation} '{anchor}')"
|
|
346
|
+
|
|
347
|
+
def vision_fn():
|
|
348
|
+
return _vision.find_element(target, anchor, relation=relation, kind=kind)
|
|
349
|
+
|
|
350
|
+
if engine == "uia":
|
|
351
|
+
return self._run_explicit(action, _uia_relation_not_impl, "uia", "uia")
|
|
352
|
+
if engine == "vision":
|
|
353
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
354
|
+
logger.info("%s: uia not implemented (DOM-Resolver folgt), using vision", action)
|
|
355
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
356
|
+
|
|
357
|
+
def tap_element(self, target: str, anchor: str, relation: str = "near",
|
|
358
|
+
kind: str = "auto", engine: str = "auto") -> None:
|
|
359
|
+
if _platform == "windows":
|
|
360
|
+
return self._tap_element_windows(target, anchor, relation, kind, engine)
|
|
361
|
+
self._run(
|
|
362
|
+
f"tap_element('{target}' {relation} '{anchor}')",
|
|
363
|
+
_uia_relation_not_impl,
|
|
364
|
+
lambda: _vision.tap_element(target, anchor, relation=relation, kind=kind),
|
|
365
|
+
engine,
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
def _tap_element_windows(self, target: str, anchor: str, relation: str, kind: str,
|
|
369
|
+
engine: str) -> None:
|
|
370
|
+
"""Windows-Dispatch für Tap Element (Cluster F), Gegenstück zu _find_element_windows,
|
|
371
|
+
gleiche Begründung."""
|
|
372
|
+
action = f"tap_element('{target}' {relation} '{anchor}')"
|
|
373
|
+
|
|
374
|
+
def vision_fn():
|
|
375
|
+
return _vision.tap_element(target, anchor, relation=relation, kind=kind)
|
|
376
|
+
|
|
377
|
+
if engine == "uia":
|
|
378
|
+
return self._run_explicit(action, _uia_relation_not_impl, "uia", "uia")
|
|
379
|
+
if engine == "vision":
|
|
380
|
+
return self._run_explicit(action, vision_fn, "vision", "vision")
|
|
381
|
+
logger.info("%s: uia not implemented (DOM-Resolver folgt), using vision", action)
|
|
382
|
+
return self._run_explicit(action, vision_fn, "vision (fallback)", "both engines")
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
import xml.etree.ElementTree as ET
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
import uiautomator2 as u2
|
|
11
|
+
|
|
12
|
+
from automation import config
|
|
13
|
+
from automation.utils import adb, scroll_centering
|
|
14
|
+
from automation.utils.cache import JsonFileCache
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
_d: u2.Device | None = None
|
|
19
|
+
_cache = JsonFileCache(config.UIA_CACHE_DIR)
|
|
20
|
+
_adb_device = adb.AdbDevice()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _screen_key() -> tuple[str, str, str, str] | None:
|
|
24
|
+
"""Stabile Screen-Identität für den UIA-Cache. None bei Fehler → Cache übersprungen."""
|
|
25
|
+
try:
|
|
26
|
+
activity = adb.current_activity()
|
|
27
|
+
if not activity or "/" not in activity:
|
|
28
|
+
return None
|
|
29
|
+
package = activity.split("/", 1)[0]
|
|
30
|
+
return package, activity, adb.app_version(package), adb.device_fingerprint()
|
|
31
|
+
except Exception:
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _parse_bounds_str(bounds_str: str) -> dict:
|
|
36
|
+
"""'[l,t][r,b]' → {'left':l, 'top':t, 'right':r, 'bottom':b} oder {} bei Parse-Fehler."""
|
|
37
|
+
nums = list(map(int, re.findall(r"-?\d+", bounds_str)))
|
|
38
|
+
if len(nums) < 4:
|
|
39
|
+
return {}
|
|
40
|
+
return {"left": nums[0], "top": nums[1], "right": nums[2], "bottom": nums[3]}
|
|
41
|
+
|
|
42
|
+
def _snapshot_text_y_and_icon_bounds(resource_id: str, text: str) -> tuple[int, list[dict]]:
|
|
43
|
+
"""Ein einziger dump_hierarchy-RPC statt N+1 .info-Roundtrips; eliminiert die Race-Condition."""
|
|
44
|
+
xml_str = _d.dump_hierarchy(compressed=True)
|
|
45
|
+
root = ET.fromstring(xml_str)
|
|
46
|
+
text_y = None
|
|
47
|
+
for node in root.iter():
|
|
48
|
+
if node.get("text") == text:
|
|
49
|
+
b = _parse_bounds_str(node.get("bounds", ""))
|
|
50
|
+
if b:
|
|
51
|
+
text_y = (b["top"] + b["bottom"]) // 2
|
|
52
|
+
break
|
|
53
|
+
if text_y is None:
|
|
54
|
+
raise RuntimeError(f"Text not found in hierarchy snapshot: '{text}'")
|
|
55
|
+
bounds_list = [
|
|
56
|
+
b for node in root.iter()
|
|
57
|
+
if node.get("resource-id") == resource_id
|
|
58
|
+
for b in [_parse_bounds_str(node.get("bounds", ""))]
|
|
59
|
+
if b
|
|
60
|
+
]
|
|
61
|
+
return text_y, bounds_list
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class UiaEngine:
|
|
65
|
+
|
|
66
|
+
def connect(self, device_id: str | None = None) -> None:
|
|
67
|
+
import uiautomator2 as u2
|
|
68
|
+
global _d
|
|
69
|
+
_d = u2.connect(device_id)
|
|
70
|
+
|
|
71
|
+
def _require_connected(self) -> None:
|
|
72
|
+
if _d is None:
|
|
73
|
+
raise RuntimeError("Device not connected. Call Connect Device first.")
|
|
74
|
+
|
|
75
|
+
def launch_app(self, package: str, activity: str | None = None) -> None:
|
|
76
|
+
self._require_connected()
|
|
77
|
+
try:
|
|
78
|
+
current_pkg = _d.app_current().get("package", "")
|
|
79
|
+
if current_pkg and current_pkg != package:
|
|
80
|
+
_d.app_stop(current_pkg)
|
|
81
|
+
except Exception:
|
|
82
|
+
pass
|
|
83
|
+
_d.app_stop(package)
|
|
84
|
+
_d.press("home")
|
|
85
|
+
_d.app_start(package, activity=activity, wait=True)
|
|
86
|
+
_d(packageName=package).wait(timeout=config.UIA_APP_START_TIMEOUT)
|
|
87
|
+
|
|
88
|
+
def text_exists(self, text: str) -> bool:
|
|
89
|
+
self._require_connected()
|
|
90
|
+
return _d(text=text).exists(timeout=config.UIA_TIMEOUT)
|
|
91
|
+
|
|
92
|
+
def tap_text(self, text: str) -> None:
|
|
93
|
+
self._require_connected()
|
|
94
|
+
el = _d(text=text)
|
|
95
|
+
if not el.exists(timeout=config.UIA_TIMEOUT):
|
|
96
|
+
raise RuntimeError(f"Text not found: '{text}'")
|
|
97
|
+
el.click()
|
|
98
|
+
|
|
99
|
+
def input_text(self, locator: str, text: str) -> None:
|
|
100
|
+
"""Feld per UIA-Locator (resourceId oder text) finden und Text setzen."""
|
|
101
|
+
self._require_connected()
|
|
102
|
+
el = _d(resourceId=locator) if ":" in locator else _d(text=locator)
|
|
103
|
+
if not el.exists(timeout=config.UIA_TIMEOUT):
|
|
104
|
+
raise RuntimeError(f"Eingabefeld nicht gefunden: '{locator}'")
|
|
105
|
+
el.set_text(text)
|
|
106
|
+
|
|
107
|
+
def tap_icon_near_text(self, icon_path: str, text: str) -> None:
|
|
108
|
+
self._require_connected()
|
|
109
|
+
resource_id = (
|
|
110
|
+
icon_path
|
|
111
|
+
if ":" in icon_path and not icon_path.endswith(".png")
|
|
112
|
+
else "android:id/icon"
|
|
113
|
+
)
|
|
114
|
+
if not _d(text=text).exists(timeout=config.UIA_TIMEOUT):
|
|
115
|
+
raise RuntimeError(f"Text not found: '{text}'")
|
|
116
|
+
text_y, bounds_list = _snapshot_text_y_and_icon_bounds(resource_id, text)
|
|
117
|
+
if not bounds_list:
|
|
118
|
+
raise RuntimeError(
|
|
119
|
+
f"No icons found via resource-id='{resource_id}'. "
|
|
120
|
+
"For apps without system icon IDs use engine='vision'."
|
|
121
|
+
)
|
|
122
|
+
icon_count = len(bounds_list)
|
|
123
|
+
closest = min(
|
|
124
|
+
range(icon_count),
|
|
125
|
+
key=lambda i: abs(
|
|
126
|
+
(bounds_list[i]["top"] + bounds_list[i]["bottom"]) // 2 - text_y
|
|
127
|
+
),
|
|
128
|
+
)
|
|
129
|
+
b = bounds_list[closest]
|
|
130
|
+
_d.click((b["left"] + b["right"]) // 2, (b["top"] + b["bottom"]) // 2)
|
|
131
|
+
|
|
132
|
+
def _center_after_scroll(self, text: str) -> bool:
|
|
133
|
+
try:
|
|
134
|
+
bounds = _d(text=text).info.get("bounds", {})
|
|
135
|
+
screen_h = _d.info.get("displayHeight", 1920)
|
|
136
|
+
screen_w = _d.info.get("displayWidth", 1080)
|
|
137
|
+
if not bounds:
|
|
138
|
+
logger.warning("_center_after_scroll: keine bounds für '%s' — Zentrierung übersprungen", text)
|
|
139
|
+
return False
|
|
140
|
+
el_center_y = (bounds.get("top", 0) + bounds.get("bottom", screen_h)) // 2
|
|
141
|
+
logger.info("_center_after_scroll el_center_y=%s screen_h=%s", el_center_y, screen_h)
|
|
142
|
+
return scroll_centering.center_in_safe_area(_adb_device, screen_w // 2, screen_h, el_center_y)
|
|
143
|
+
except Exception as e:
|
|
144
|
+
logger.warning("_center_after_scroll failed: %s: %s", type(e).__name__, e)
|
|
145
|
+
return False
|
|
146
|
+
|
|
147
|
+
def scroll_to_text(self, text: str) -> bool:
|
|
148
|
+
self._require_connected()
|
|
149
|
+
|
|
150
|
+
# Kein Scroll nötig wenn Text bereits sichtbar (schützt auch gegen Cache-Vergiftung mit k=0)
|
|
151
|
+
if _d(text=text).exists(timeout=config.UIA_TIMEOUT):
|
|
152
|
+
return True
|
|
153
|
+
|
|
154
|
+
# Pfad 1: Cache-Hit (nur wenn UIA_DUMP_CACHE)
|
|
155
|
+
if config.UIA_DUMP_CACHE:
|
|
156
|
+
try:
|
|
157
|
+
key = _screen_key()
|
|
158
|
+
if key:
|
|
159
|
+
package, activity, version, device_fp = key
|
|
160
|
+
action_key = f"scroll_to_text|text={text}|direction=down"
|
|
161
|
+
hint = _cache.get_uia_hint(package, activity, version, device_fp, action_key)
|
|
162
|
+
if hint is not None:
|
|
163
|
+
k = hint.get("swipes_k", 0)
|
|
164
|
+
scroller = _d(scrollable=True)
|
|
165
|
+
for _ in range(k):
|
|
166
|
+
scroller.scroll.forward()
|
|
167
|
+
# bis zu 2 Extra-Swipes für Off-by-One
|
|
168
|
+
for extra in range(3):
|
|
169
|
+
if _d(text=text).exists(timeout=config.UIA_TIMEOUT):
|
|
170
|
+
logger.info("UIA_CACHE event=hit keyword=scroll_to_text target=%r swipes_k=%d", text, k + extra)
|
|
171
|
+
centered = self._center_after_scroll(text)
|
|
172
|
+
if not centered:
|
|
173
|
+
_d.jsonrpc.waitForIdle(int(config.UIA_WAIT_IDLE_TIMEOUT * 1000))
|
|
174
|
+
return True
|
|
175
|
+
scroller.scroll.forward()
|
|
176
|
+
logger.info("UIA_CACHE event=verify_fail keyword=scroll_to_text target=%r swipes_k=%d", text, k)
|
|
177
|
+
except Exception as e:
|
|
178
|
+
logger.warning("UIA cache lookup failed: %s", e)
|
|
179
|
+
|
|
180
|
+
# Pfad 2: Diskrete Lernschleife (primär, Cache-Miss)
|
|
181
|
+
scroller = _d(scrollable=True)
|
|
182
|
+
for swipe_idx in range(1, config.UIA_SCROLL_MAX_SWIPES + 1):
|
|
183
|
+
try:
|
|
184
|
+
scroller.scroll.forward()
|
|
185
|
+
except Exception:
|
|
186
|
+
break
|
|
187
|
+
if _d(text=text).exists(timeout=1):
|
|
188
|
+
if config.UIA_DUMP_CACHE:
|
|
189
|
+
try:
|
|
190
|
+
key = _screen_key()
|
|
191
|
+
if key:
|
|
192
|
+
package, activity, version, device_fp = key
|
|
193
|
+
action_key = f"scroll_to_text|text={text}|direction=down"
|
|
194
|
+
_cache.put_uia_hint(package, activity, version, device_fp, action_key,
|
|
195
|
+
{"swipes_k": swipe_idx})
|
|
196
|
+
logger.info("UIA_CACHE event=miss keyword=scroll_to_text target=%r swipes_k=%d", text, swipe_idx)
|
|
197
|
+
except Exception as e:
|
|
198
|
+
logger.warning("UIA cache write failed: %s", e)
|
|
199
|
+
centered = self._center_after_scroll(text)
|
|
200
|
+
if not centered:
|
|
201
|
+
_d.jsonrpc.waitForIdle(int(config.UIA_WAIT_IDLE_TIMEOUT * 1000))
|
|
202
|
+
return True
|
|
203
|
+
|
|
204
|
+
# Pfad 3: Letzter Notausgang — original scroll.to
|
|
205
|
+
for attempt in range(1, config.SCROLL_RETRIES + 1):
|
|
206
|
+
try:
|
|
207
|
+
t0 = time.monotonic()
|
|
208
|
+
result = _d(scrollable=True).scroll.to(text=text)
|
|
209
|
+
elapsed_ms = int((time.monotonic() - t0) * 1000)
|
|
210
|
+
logger.info("scroll attempt=%s result=%s elapsed_ms=%s", attempt, result, elapsed_ms)
|
|
211
|
+
if result:
|
|
212
|
+
centered = self._center_after_scroll(text)
|
|
213
|
+
if not centered:
|
|
214
|
+
_d.jsonrpc.waitForIdle(int(config.UIA_WAIT_IDLE_TIMEOUT * 1000))
|
|
215
|
+
return True
|
|
216
|
+
except Exception as e:
|
|
217
|
+
elapsed_ms = int((time.monotonic() - t0) * 1000)
|
|
218
|
+
logger.warning("scroll attempt=%s failed after %sms: %s: %s",
|
|
219
|
+
attempt, elapsed_ms, type(e).__name__, e)
|
|
220
|
+
return _d(text=text).exists(timeout=config.UIA_SCROLL_TIMEOUT)
|