dynamic-test-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- automation/__init__.py +0 -0
- automation/config.py +69 -0
- automation/engines/__init__.py +0 -0
- automation/engines/engine_manager.py +382 -0
- automation/engines/uia_engine.py +220 -0
- automation/engines/vision_base.py +30 -0
- automation/engines/vision_engine.py +474 -0
- automation/engines/windows_uia_engine.py +347 -0
- automation/keywords/__init__.py +0 -0
- automation/keywords/automation_keywords.py +83 -0
- automation/utils/__init__.py +0 -0
- automation/utils/adb.py +220 -0
- automation/utils/cache.py +135 -0
- automation/utils/device_interface.py +60 -0
- automation/utils/image_processing.py +258 -0
- automation/utils/scroll_centering.py +48 -0
- automation/utils/spatial.py +63 -0
- automation/utils/windows_host.py +354 -0
- dynamic_test_engine-0.1.0.dist-info/METADATA +405 -0
- dynamic_test_engine-0.1.0.dist-info/RECORD +23 -0
- dynamic_test_engine-0.1.0.dist-info/WHEEL +5 -0
- dynamic_test_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
- dynamic_test_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import glob
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
|
|
7
|
+
from automation import config
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
class JsonFileCache:
|
|
12
|
+
|
|
13
|
+
def __init__(self, cache_dir: str | None = None) -> None:
|
|
14
|
+
self._dir = cache_dir or config.UIA_CACHE_DIR
|
|
15
|
+
|
|
16
|
+
def _path(self, package: str) -> str:
|
|
17
|
+
os.makedirs(self._dir, exist_ok=True)
|
|
18
|
+
safe = package.replace(":", "_").replace("/", "_")
|
|
19
|
+
return os.path.join(self._dir, f"{safe}.json")
|
|
20
|
+
|
|
21
|
+
@staticmethod
|
|
22
|
+
def _screen_key(activity: str, version: str, device_fp: str) -> str:
|
|
23
|
+
return f"{activity}|{version}|{device_fp}"
|
|
24
|
+
|
|
25
|
+
def _load_screens(self, package: str) -> dict:
|
|
26
|
+
path = self._path(package)
|
|
27
|
+
if not os.path.exists(path):
|
|
28
|
+
return {}
|
|
29
|
+
try:
|
|
30
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
31
|
+
data = json.load(f)
|
|
32
|
+
except (json.JSONDecodeError, OSError) as e:
|
|
33
|
+
logger.warning("cache: Laden fehlgeschlagen, Datei wird als leer behandelt path=%r error=%s", path, e)
|
|
34
|
+
return {}
|
|
35
|
+
return data.get("screens", {})
|
|
36
|
+
|
|
37
|
+
def get(self, package: str, activity: str, version: str, device_fp: str) -> dict | None:
|
|
38
|
+
screens = self._load_screens(package)
|
|
39
|
+
return screens.get(self._screen_key(activity, version, device_fp))
|
|
40
|
+
|
|
41
|
+
def put(self, package: str, activity: str, version: str, device_fp: str,
|
|
42
|
+
elements, index=None, framework_version: str = "") -> None:
|
|
43
|
+
screens = self._load_screens(package)
|
|
44
|
+
# Versions-Invalidierung: Einträge einer anderen App-Version derselben Package verwerfen.
|
|
45
|
+
screens = {k: v for k, v in screens.items() if v.get("version") == version}
|
|
46
|
+
# Merge statt Überschreiben: vorhandenes custom_elements-Feld bleibt erhalten (sonst löscht ein OCR-Miss den Icon-Cache).
|
|
47
|
+
entry = screens.setdefault(self._screen_key(activity, version, device_fp), {})
|
|
48
|
+
entry.update({
|
|
49
|
+
"activity": activity,
|
|
50
|
+
"version": version,
|
|
51
|
+
"device_fingerprint": device_fp,
|
|
52
|
+
"elements": elements,
|
|
53
|
+
})
|
|
54
|
+
# index/framework_version nur überschreiben, wenn der Aufruf sie liefert (Vision nie ->
|
|
55
|
+
# vom UIA-Index-Writer geschriebene Werte bleiben erhalten).
|
|
56
|
+
if framework_version:
|
|
57
|
+
entry["framework_version"] = framework_version
|
|
58
|
+
else:
|
|
59
|
+
entry.setdefault("framework_version", "")
|
|
60
|
+
if index is not None:
|
|
61
|
+
entry["index"] = index
|
|
62
|
+
else:
|
|
63
|
+
entry.setdefault("index", {})
|
|
64
|
+
_atomic_write(self._path(package), {
|
|
65
|
+
"schema_version": "2",
|
|
66
|
+
"package": package,
|
|
67
|
+
"screens": screens,
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
def get_custom_element_matches(self, package: str, activity: str, version: str,
|
|
71
|
+
device_fp: str, element_path: str) -> list[dict] | None:
|
|
72
|
+
entry = self.get(package, activity, version, device_fp)
|
|
73
|
+
if entry is None:
|
|
74
|
+
return None
|
|
75
|
+
return entry.get("custom_elements", {}).get(element_path)
|
|
76
|
+
|
|
77
|
+
def put_custom_element_matches(self, package: str, activity: str, version: str,
|
|
78
|
+
device_fp: str, element_path: str, boxes: list[dict]) -> None:
|
|
79
|
+
screens = self._load_screens(package)
|
|
80
|
+
# Versions-Invalidierung wie in put (sonst wächst die Datei über App-Versionen hinweg an).
|
|
81
|
+
screens = {k: v for k, v in screens.items() if v.get("version") == version}
|
|
82
|
+
key = self._screen_key(activity, version, device_fp)
|
|
83
|
+
# Minimaleintrag, falls der Screen-Key noch kein OCR-Layout hat; custom_elements wird
|
|
84
|
+
# ergänzt ohne elements zu berühren.
|
|
85
|
+
entry = screens.setdefault(key, {
|
|
86
|
+
"activity": activity, "version": version,
|
|
87
|
+
"device_fingerprint": device_fp, "framework_version": "",
|
|
88
|
+
"elements": [], "index": {},
|
|
89
|
+
})
|
|
90
|
+
entry.setdefault("custom_elements", {})[element_path] = boxes
|
|
91
|
+
_atomic_write(self._path(package), {
|
|
92
|
+
"schema_version": "2", "package": package, "screens": screens,
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
def get_uia_hint(self, package: str, activity: str, version: str,
|
|
96
|
+
device_fp: str, action_key: str) -> dict | None:
|
|
97
|
+
entry = self.get(package, activity, version, device_fp)
|
|
98
|
+
if entry is None:
|
|
99
|
+
return None
|
|
100
|
+
return entry.get("uia_hints", {}).get(action_key)
|
|
101
|
+
|
|
102
|
+
def put_uia_hint(self, package: str, activity: str, version: str,
|
|
103
|
+
device_fp: str, action_key: str, hint: dict) -> None:
|
|
104
|
+
screens = self._load_screens(package)
|
|
105
|
+
screens = {k: v for k, v in screens.items() if v.get("version") == version}
|
|
106
|
+
key = self._screen_key(activity, version, device_fp)
|
|
107
|
+
entry = screens.setdefault(key, {
|
|
108
|
+
"activity": activity, "version": version,
|
|
109
|
+
"device_fingerprint": device_fp, "framework_version": "",
|
|
110
|
+
"elements": [], "index": {},
|
|
111
|
+
})
|
|
112
|
+
entry.setdefault("uia_hints", {})[action_key] = hint
|
|
113
|
+
_atomic_write(self._path(package), {
|
|
114
|
+
"schema_version": "2", "package": package, "screens": screens,
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
def invalidate(self, package: str | None = None) -> None:
|
|
118
|
+
if package is not None:
|
|
119
|
+
path = self._path(package)
|
|
120
|
+
if os.path.exists(path):
|
|
121
|
+
os.remove(path)
|
|
122
|
+
logger.info("cache: invalidate package=%r", package)
|
|
123
|
+
return
|
|
124
|
+
paths = glob.glob(os.path.join(self._dir, "*.json"))
|
|
125
|
+
for path in paths:
|
|
126
|
+
os.remove(path)
|
|
127
|
+
logger.info("cache: invalidate package=all deleted=%d", len(paths))
|
|
128
|
+
|
|
129
|
+
def _atomic_write(path: str, data: dict) -> None:
|
|
130
|
+
dir_ = os.path.dirname(os.path.abspath(path))
|
|
131
|
+
with tempfile.NamedTemporaryFile("w", dir=dir_, delete=False,
|
|
132
|
+
suffix=".tmp", encoding="utf-8") as f:
|
|
133
|
+
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
134
|
+
tmp = f.name
|
|
135
|
+
os.replace(tmp, path)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DeviceInterface(ABC):
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def screenshot(self) -> np.ndarray: ...
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def tap(self, x: int, y: int) -> None: ...
|
|
17
|
+
|
|
18
|
+
@abstractmethod
|
|
19
|
+
def type_text(self, text: str) -> None:
|
|
20
|
+
"""Tippt literalen Text in das aktuell fokussierte Element.
|
|
21
|
+
PoC: nur Leerzeichen werden behandelt, kein Sonderzeichen-Escaping."""
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def scroll(self, x: int, y: int, amount: int, duration_ms: int | None = None) -> None:
|
|
26
|
+
"""Scrollt die Ansicht unter (x, y) um `amount` (Vorzeichen = Richtung). Plattformspezifische
|
|
27
|
+
Geste: Touch-Drag auf Android, Mausrad auf Windows."""
|
|
28
|
+
...
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def connect(self, target: str | None = None) -> None: ...
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def screen_key(self) -> tuple[str, str, str, str] | None:
|
|
35
|
+
"""Stabile Screen-Identität (package/prozess, activity/fenster, version, device_fp).
|
|
36
|
+
None bei jedem Fehler -> Cache übersprungen."""
|
|
37
|
+
...
|
|
38
|
+
|
|
39
|
+
def move_mouse(self, x: int, y: int) -> None:
|
|
40
|
+
"""Bewegt den Mauszeiger an (x, y) ohne zu klicken. Nur auf Plattformen mit
|
|
41
|
+
Zeigerkonzept (Windows) implementiert."""
|
|
42
|
+
raise NotImplementedError(f"{type(self).__name__} unterstützt move_mouse nicht.")
|
|
43
|
+
|
|
44
|
+
def verify_focus_in_target(self) -> bool:
|
|
45
|
+
"""Prüft, ob der Eingabefokus im verbundenen Zielprozess liegt. Default True."""
|
|
46
|
+
return True
|
|
47
|
+
|
|
48
|
+
def active_region(self) -> tuple[int, int, int, int] | None:
|
|
49
|
+
"""Ausdehnung des obersten Dialog-/Fenster-Overlays in Screenshot-relativen Koordinaten,
|
|
50
|
+
None ohne Overlay. Nur auf Plattformen mit Overlay-Fensterkonzept implementiert."""
|
|
51
|
+
raise NotImplementedError(f"{type(self).__name__} unterstützt active_region nicht.")
|
|
52
|
+
|
|
53
|
+
def value_region(self, text: str) -> tuple[int, int, int, int] | None:
|
|
54
|
+
"""Ausdehnung des einen editierbaren Elements mit aktuellem Wert == text, None bei
|
|
55
|
+
0/>1 Treffern. Nur auf Windows implementiert."""
|
|
56
|
+
raise NotImplementedError(f"{type(self).__name__} unterstützt value_region nicht.")
|
|
57
|
+
|
|
58
|
+
def clear_focused_field(self, x: int, y: int) -> None:
|
|
59
|
+
"""Leert das Eingabefeld an Position (x, y) vor der Texteingabe."""
|
|
60
|
+
raise NotImplementedError(f"{type(self).__name__} unterstützt clear_focused_field nicht.")
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import platform
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from automation import config
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_image(path: str) -> np.ndarray:
|
|
17
|
+
import cv2
|
|
18
|
+
img = cv2.imread(path)
|
|
19
|
+
if img is None:
|
|
20
|
+
raise FileNotFoundError(f"load_image: image not found: {path}")
|
|
21
|
+
return img
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def to_grayscale(image: np.ndarray) -> np.ndarray:
|
|
25
|
+
import cv2
|
|
26
|
+
return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def match_custom_element(
|
|
30
|
+
screenshot: np.ndarray,
|
|
31
|
+
custom_element: np.ndarray,
|
|
32
|
+
threshold: float,
|
|
33
|
+
) -> list[tuple[int, int]]:
|
|
34
|
+
import cv2
|
|
35
|
+
import numpy as np
|
|
36
|
+
gray_screen = to_grayscale(screenshot)
|
|
37
|
+
gray_element = to_grayscale(custom_element)
|
|
38
|
+
result = cv2.matchTemplate(gray_screen, gray_element, cv2.TM_CCOEFF_NORMED)
|
|
39
|
+
locations = np.where(result >= threshold)
|
|
40
|
+
h, w = gray_element.shape
|
|
41
|
+
matches = [(int(x + w // 2), int(y + h // 2)) for x, y in zip(locations[1], locations[0])]
|
|
42
|
+
logger.debug("match_custom_element: threshold=%s matches=%s", threshold, len(matches))
|
|
43
|
+
return matches
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def filter_matches(
|
|
47
|
+
matches: list[tuple[int, int]],
|
|
48
|
+
min_distance: int = 20,
|
|
49
|
+
) -> list[tuple[int, int]]:
|
|
50
|
+
filtered = []
|
|
51
|
+
for candidate in matches:
|
|
52
|
+
too_close = any(
|
|
53
|
+
abs(candidate[0] - kept[0]) < min_distance
|
|
54
|
+
and abs(candidate[1] - kept[1]) < min_distance
|
|
55
|
+
for kept in filtered
|
|
56
|
+
)
|
|
57
|
+
if not too_close:
|
|
58
|
+
filtered.append(candidate)
|
|
59
|
+
logger.debug("filter_matches: input=%s min_distance=%s filtered=%s", len(matches), min_distance, len(filtered))
|
|
60
|
+
return filtered
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _maybe_invert(image: np.ndarray) -> np.ndarray:
|
|
64
|
+
import cv2
|
|
65
|
+
import numpy as np
|
|
66
|
+
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
|
67
|
+
if float(np.mean(gray)) < 100:
|
|
68
|
+
return cv2.bitwise_not(image)
|
|
69
|
+
return image
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _find_dark_regions(gray: np.ndarray, min_area: int = 500) -> list[tuple[int, int, int, int]]:
|
|
73
|
+
"""Begrenzungsrechtecke einzelner dunkler Blöcke in einem ansonsten hellen Bild.
|
|
74
|
+
MORPH_OPEN statt CLOSE, damit normaler schwarzer Text nicht mit zu Blöcken verschmilzt."""
|
|
75
|
+
import cv2
|
|
76
|
+
import numpy as np
|
|
77
|
+
mask = (gray < 100).astype("uint8") * 255
|
|
78
|
+
kernel = np.ones((9, 9), np.uint8)
|
|
79
|
+
opened = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
|
|
80
|
+
contours, _ = cv2.findContours(opened, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
81
|
+
boxes = [cv2.boundingRect(c) for c in contours]
|
|
82
|
+
return [b for b in boxes if b[2] * b[3] >= min_area]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _merge_new_tokens(results: list[dict], new_tokens: list[dict], dx: int = 0, dy: int = 0) -> None:
|
|
86
|
+
"""Fügt new_tokens (optional um dx/dy verschoben) hinzu. Ersetzt dabei überlappende
|
|
87
|
+
ältere Boxen, die im neuen Token aufgehen, statt am Overlap zu scheitern."""
|
|
88
|
+
margin = 5
|
|
89
|
+
for tok in new_tokens:
|
|
90
|
+
left = tok["left"] + dx
|
|
91
|
+
top = tok["top"] + dy
|
|
92
|
+
box = (left - margin, top - margin, left + tok["width"] + margin, top + tok["height"] + margin)
|
|
93
|
+
new_area = tok["width"] * tok["height"]
|
|
94
|
+
superseded, is_duplicate = _resolve_overlaps(results, box, new_area)
|
|
95
|
+
if is_duplicate:
|
|
96
|
+
continue
|
|
97
|
+
for r in superseded:
|
|
98
|
+
results.remove(r)
|
|
99
|
+
tok["left"], tok["top"] = left, top
|
|
100
|
+
results.append(tok)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _resolve_overlaps(
|
|
104
|
+
results: list[dict], box: tuple[int, int, int, int], new_area: int,
|
|
105
|
+
) -> tuple[list[dict], bool]:
|
|
106
|
+
"""Prüft box gegen bestehende results. superseded sind Boxen, die box vollständig ersetzt,
|
|
107
|
+
is_duplicate=True wenn box selbst in einer bestehenden Box aufgeht."""
|
|
108
|
+
bx0, by0, bx1, by1 = box
|
|
109
|
+
superseded = []
|
|
110
|
+
for r in results:
|
|
111
|
+
ox0, oy0 = max(bx0, r["left"]), max(by0, r["top"])
|
|
112
|
+
ox1, oy1 = min(bx1, r["left"] + r["width"]), min(by1, r["top"] + r["height"])
|
|
113
|
+
if ox0 >= ox1 or oy0 >= oy1:
|
|
114
|
+
continue
|
|
115
|
+
overlap_area = (ox1 - ox0) * (oy1 - oy0)
|
|
116
|
+
r_area = r["width"] * r["height"]
|
|
117
|
+
if r_area > 0 and overlap_area / r_area >= 0.8 and new_area > r_area:
|
|
118
|
+
superseded.append(r)
|
|
119
|
+
else:
|
|
120
|
+
return superseded, True
|
|
121
|
+
return superseded, False
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def ocr_find_text(
|
|
125
|
+
image: np.ndarray,
|
|
126
|
+
lang: str = "deu+eng",
|
|
127
|
+
scale: float | None = None,
|
|
128
|
+
) -> list[dict]:
|
|
129
|
+
import cv2
|
|
130
|
+
import numpy as np
|
|
131
|
+
import pytesseract
|
|
132
|
+
if platform.system() == "Windows":
|
|
133
|
+
_venv_tess = Path(__file__).parents[2] / ".venv" / "Scripts" / "tesseract.exe"
|
|
134
|
+
if _venv_tess.exists():
|
|
135
|
+
pytesseract.pytesseract.tesseract_cmd = str(_venv_tess)
|
|
136
|
+
scale = scale or config.OCR_SCALE
|
|
137
|
+
if scale <= 0:
|
|
138
|
+
scale = 1.0
|
|
139
|
+
|
|
140
|
+
def _run_ocr(
|
|
141
|
+
src: np.ndarray,
|
|
142
|
+
block_offset: int = 0,
|
|
143
|
+
psm: int | None = None,
|
|
144
|
+
low_conf_sink: list[dict] | None = None,
|
|
145
|
+
) -> list[dict]:
|
|
146
|
+
proc = src
|
|
147
|
+
if scale != 1.0:
|
|
148
|
+
h, w = proc.shape[:2]
|
|
149
|
+
proc = cv2.resize(proc, (int(w * scale), int(h * scale)))
|
|
150
|
+
_psm = psm if psm is not None else config.OCR_PSM
|
|
151
|
+
custom_config = f"--psm {_psm} --oem {config.OCR_OEM}"
|
|
152
|
+
data = pytesseract.image_to_data(proc, lang=lang, config=custom_config, output_type=pytesseract.Output.DICT)
|
|
153
|
+
out = []
|
|
154
|
+
for i, text in enumerate(data["text"]):
|
|
155
|
+
if not text.strip():
|
|
156
|
+
continue
|
|
157
|
+
conf = int(data["conf"][i])
|
|
158
|
+
token = {
|
|
159
|
+
"text": text.strip(),
|
|
160
|
+
"left": int(data["left"][i] / scale),
|
|
161
|
+
"top": int(data["top"][i] / scale),
|
|
162
|
+
"width": int(data["width"][i] / scale),
|
|
163
|
+
"height": int(data["height"][i] / scale),
|
|
164
|
+
"block_num": int(data["block_num"][i]) + block_offset,
|
|
165
|
+
"par_num": int(data["par_num"][i]) + block_offset,
|
|
166
|
+
"line_num": int(data["line_num"][i]),
|
|
167
|
+
}
|
|
168
|
+
if conf > config.OCR_CONFIDENCE:
|
|
169
|
+
out.append(token)
|
|
170
|
+
elif conf > 0 and low_conf_sink is not None:
|
|
171
|
+
low_conf_sink.append(token)
|
|
172
|
+
return out
|
|
173
|
+
|
|
174
|
+
def _crop_ocr_merge(
|
|
175
|
+
results: list[dict],
|
|
176
|
+
x: int, y: int, w: int, h: int,
|
|
177
|
+
pad_x: int, pad_y: int,
|
|
178
|
+
block_offset: int, psm: int,
|
|
179
|
+
transform=None, downscale: int = 1,
|
|
180
|
+
) -> None:
|
|
181
|
+
"""Croppt einen Bereich (plus Padding), OCR't ihn separat und mergt die Treffer zurück
|
|
182
|
+
in results. Gemeinsame Grundlage für Dark-Region- und Low-Conf-Rescue-Pass."""
|
|
183
|
+
x0, x1 = max(0, x - pad_x), min(image.shape[1], x + w + pad_x)
|
|
184
|
+
y0, y1 = max(0, y - pad_y), min(image.shape[0], y + h + pad_y)
|
|
185
|
+
crop = image[y0:y1, x0:x1]
|
|
186
|
+
if crop.size == 0:
|
|
187
|
+
return
|
|
188
|
+
if transform is not None:
|
|
189
|
+
crop = transform(crop)
|
|
190
|
+
tokens = _run_ocr(crop, block_offset=block_offset, psm=psm)
|
|
191
|
+
if downscale != 1:
|
|
192
|
+
for tok in tokens:
|
|
193
|
+
tok["left"] //= downscale
|
|
194
|
+
tok["top"] //= downscale
|
|
195
|
+
tok["width"] //= downscale
|
|
196
|
+
tok["height"] //= downscale
|
|
197
|
+
_merge_new_tokens(results, tokens, dx=x0, dy=y0)
|
|
198
|
+
|
|
199
|
+
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
|
200
|
+
is_dark = float(np.mean(gray)) < 100
|
|
201
|
+
low_conf_tokens: list[dict] = []
|
|
202
|
+
results = _run_ocr(_maybe_invert(image), low_conf_sink=low_conf_tokens)
|
|
203
|
+
|
|
204
|
+
if is_dark:
|
|
205
|
+
_merge_new_tokens(results, _run_ocr(image, block_offset=1000, psm=3))
|
|
206
|
+
else:
|
|
207
|
+
pad = 6
|
|
208
|
+
for i, (x, y, w, h) in enumerate(_find_dark_regions(gray)):
|
|
209
|
+
_crop_ocr_merge(
|
|
210
|
+
results, x, y, w, h, pad_x=pad, pad_y=pad,
|
|
211
|
+
block_offset=2000 + i * 100, psm=6, transform=cv2.bitwise_not,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
# Adaptive-Upscale-Rescue: Tokens, die Tesseract erkannt, aber nicht sicher gelesen hat
|
|
215
|
+
# (z. B. abgeschnittene CamelCase-Compound-Wörter wie "CollectionNavigator"), sind ein
|
|
216
|
+
# generischer Hinweis auf zu geringe Zeichenauflösung an dieser Stelle - unabhängig von
|
|
217
|
+
# Schriftgröße, da dieselbe Bildschirmschrift an anderer Stelle sauber gelesen wird.
|
|
218
|
+
# Einzelzeichen-Fragmente (Icon-Glyphen, Ziffern) werden ausgeklammert, da Upscaling dort
|
|
219
|
+
# nichts rekonstruiert, was nicht schon vorhanden ist.
|
|
220
|
+
upscale = 3
|
|
221
|
+
for i, tok in enumerate(low_conf_tokens):
|
|
222
|
+
if len(tok["text"]) < 2:
|
|
223
|
+
continue
|
|
224
|
+
left, t, w, h = tok["left"], tok["top"], tok["width"], tok["height"]
|
|
225
|
+
pad_x, pad_y = max(40, h * 6), max(15, h * 2)
|
|
226
|
+
_crop_ocr_merge(
|
|
227
|
+
results, left, t, w, h, pad_x=pad_x, pad_y=pad_y,
|
|
228
|
+
block_offset=3000 + i * 100, psm=3,
|
|
229
|
+
transform=lambda c: cv2.resize(c, None, fx=upscale, fy=upscale, interpolation=cv2.INTER_CUBIC),
|
|
230
|
+
downscale=upscale,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
logger.debug(
|
|
234
|
+
"ocr_find_text: lang=%s scale=%s results=%s low_conf_rescued=%s",
|
|
235
|
+
lang, scale, len(results), len(low_conf_tokens),
|
|
236
|
+
)
|
|
237
|
+
return results
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _merge_line(line: list[dict]) -> dict:
|
|
241
|
+
"""Fasst die Tokens einer OCR-Zeile zu einer Bounding-Box mit zusammengefügtem Text zusammen."""
|
|
242
|
+
words, x0, y0, x1, y1 = [], line[0]["left"], line[0]["top"], 0, 0
|
|
243
|
+
for t in line:
|
|
244
|
+
words.append(t["text"])
|
|
245
|
+
y0 = min(y0, t["top"])
|
|
246
|
+
x1 = max(x1, t["left"] + t["width"])
|
|
247
|
+
y1 = max(y1, t["top"] + t["height"])
|
|
248
|
+
return {"text": " ".join(words), "left": x0, "top": y0, "width": x1 - x0, "height": y1 - y0}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def ocr_group_lines(tokens: list[dict]) -> list[dict]:
|
|
252
|
+
# Erwartet Tokens mit den Schlüsseln: block_num, par_num, line_num, left, top, width, height, text.
|
|
253
|
+
if not tokens:
|
|
254
|
+
return []
|
|
255
|
+
groups: dict[tuple, list[dict]] = {}
|
|
256
|
+
for t in tokens:
|
|
257
|
+
groups.setdefault((t["block_num"], t["par_num"], t["line_num"]), []).append(t)
|
|
258
|
+
return [_merge_line(sorted(groups[key], key=lambda t: t["left"])) for key in sorted(groups)]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
from automation import config
|
|
7
|
+
from automation.utils.device_interface import DeviceInterface
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def center_in_safe_area(
|
|
13
|
+
device: DeviceInterface,
|
|
14
|
+
x: int,
|
|
15
|
+
screen_height: int,
|
|
16
|
+
el_center_y: int,
|
|
17
|
+
margin_percent: float | None = None,
|
|
18
|
+
max_shift: int | None = None,
|
|
19
|
+
) -> bool:
|
|
20
|
+
"""Verschiebt ein per Scroll gefundenes Element minimal in den sicheren Bereich, falls es
|
|
21
|
+
außerhalb liegt. Gibt zurück, ob eine Korrektur nötig war."""
|
|
22
|
+
if margin_percent is None:
|
|
23
|
+
margin_percent = config.SCROLL_SAFE_MARGIN_PERCENT
|
|
24
|
+
|
|
25
|
+
top = screen_height * margin_percent
|
|
26
|
+
bottom = screen_height * (1 - margin_percent)
|
|
27
|
+
if top <= el_center_y <= bottom:
|
|
28
|
+
return False # bereits im sicheren Bereich, keine Korrektur nötig
|
|
29
|
+
|
|
30
|
+
# Ziel = nächste Bereichsgrenze → minimale Bewegung, kann nie über die Mitte hinausschießen
|
|
31
|
+
target_y = bottom if el_center_y > bottom else top
|
|
32
|
+
delta = int(el_center_y - target_y) # >0: Element nach oben holen; <0: nach unten
|
|
33
|
+
if max_shift is not None:
|
|
34
|
+
delta = max(-max_shift, min(delta, max_shift))
|
|
35
|
+
# Unterhalb dieser Schwelle ist ein adb-Swipe vom Gerät nicht von einem Tap zu unterscheiden.
|
|
36
|
+
# Mikrokorrekturen überspringen, das Element ist nah genug an der Bereichsgrenze.
|
|
37
|
+
if abs(delta) < 30:
|
|
38
|
+
return False
|
|
39
|
+
|
|
40
|
+
# Scroll-Distanz = |delta| (exakte Korrektur), zentriert um den Bildschirm-Mittelpunkt,
|
|
41
|
+
# damit Start- und Endkoordinaten sicher im sichtbaren Bereich bleiben.
|
|
42
|
+
mid = screen_height // 2
|
|
43
|
+
logger.info(
|
|
44
|
+
"center_in_safe_area el_center_y=%s target_y=%s delta=%s", el_center_y, int(target_y), delta
|
|
45
|
+
)
|
|
46
|
+
device.scroll(x, mid, delta, duration_ms=config.CENTERING_SWIPE_DURATION_MS)
|
|
47
|
+
time.sleep(0.2) # Korrektur-Swipe abwarten, bevor der Aufrufer den Bildschirmzustand liest
|
|
48
|
+
return True
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Geometrischer Relations-Resolver (engine-unabhängig), arbeitet rein auf Bounding-Boxes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
RELATIONS = ("near", "left_of", "right_of", "above", "below")
|
|
10
|
+
|
|
11
|
+
def center(box: dict) -> tuple[int, int]:
|
|
12
|
+
return box["left"] + box["width"] // 2, box["top"] + box["height"] // 2
|
|
13
|
+
|
|
14
|
+
def _rows_overlap(a: dict, b: dict, tol: int) -> bool:
|
|
15
|
+
# Vertikale Bereiche überlappen sich (gleiche Zeile), mit Toleranz.
|
|
16
|
+
return a["top"] < b["top"] + b["height"] + tol and b["top"] < a["top"] + a["height"] + tol
|
|
17
|
+
|
|
18
|
+
def _cols_overlap(a: dict, b: dict, tol: int) -> bool:
|
|
19
|
+
# Horizontale Bereiche überlappen sich (gleiche Spalte), mit Toleranz.
|
|
20
|
+
return a["left"] < b["left"] + b["width"] + tol and b["left"] < a["left"] + a["width"] + tol
|
|
21
|
+
|
|
22
|
+
def _distance(a: dict, b: dict) -> float:
|
|
23
|
+
ax, ay = center(a)
|
|
24
|
+
bx, by = center(b)
|
|
25
|
+
return ((ax - bx) ** 2 + (ay - by) ** 2) ** 0.5
|
|
26
|
+
|
|
27
|
+
def resolve(
|
|
28
|
+
anchor: dict,
|
|
29
|
+
candidates: list[dict],
|
|
30
|
+
relation: str,
|
|
31
|
+
*,
|
|
32
|
+
overlap_tol: int = 0,
|
|
33
|
+
max_distance: int = 0,
|
|
34
|
+
) -> dict | None:
|
|
35
|
+
"""Gibt die Kandidaten-Box zurück, die die Relation zum Anker erfüllt und am nächsten liegt,
|
|
36
|
+
None wenn kein Kandidat passt oder max_distance überschritten wird."""
|
|
37
|
+
if relation not in RELATIONS:
|
|
38
|
+
raise ValueError(f"Unbekannte Relation: '{relation}'. Erlaubt: {RELATIONS}")
|
|
39
|
+
acx, acy = center(anchor)
|
|
40
|
+
matching = []
|
|
41
|
+
for c in candidates:
|
|
42
|
+
ccx, ccy = center(c)
|
|
43
|
+
if relation == "near":
|
|
44
|
+
ok = True
|
|
45
|
+
elif relation == "left_of":
|
|
46
|
+
ok = ccx < acx and _rows_overlap(anchor, c, overlap_tol)
|
|
47
|
+
elif relation == "right_of":
|
|
48
|
+
ok = ccx > acx and _rows_overlap(anchor, c, overlap_tol)
|
|
49
|
+
elif relation == "above":
|
|
50
|
+
ok = ccy < acy and _cols_overlap(anchor, c, overlap_tol)
|
|
51
|
+
else: # below
|
|
52
|
+
ok = ccy > acy and _cols_overlap(anchor, c, overlap_tol)
|
|
53
|
+
if ok:
|
|
54
|
+
matching.append(c)
|
|
55
|
+
if not matching:
|
|
56
|
+
logger.debug("resolve event=no_match relation=%s candidates=%d", relation, len(candidates))
|
|
57
|
+
return None
|
|
58
|
+
best = min(matching, key=lambda c: _distance(anchor, c))
|
|
59
|
+
if max_distance and _distance(anchor, best) > max_distance:
|
|
60
|
+
logger.debug("resolve event=max_distance_exceeded relation=%s distance=%.1f max_distance=%d",
|
|
61
|
+
relation, _distance(anchor, best), max_distance)
|
|
62
|
+
return None
|
|
63
|
+
return best
|