dynamic-test-engine 0.1.0__py3-none-any.whl

automation/engines/vision_base.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import numpy as np
+
+class VisionEngineBase(ABC):
+
+    @abstractmethod
+    def text_exists(self, text: str, img: "np.ndarray | None" = None) -> bool: ...
+
+    @abstractmethod
+    def tap_text(self, text: str) -> None: ...
+
+    @abstractmethod
+    def tap_icon_near_text(self, icon_path: str, text: str) -> None: ...
+
+    @abstractmethod
+    def scroll_to_text(self, text: str, anchor: "str | None" = None, relation: str = "near") -> bool: ...
+
+    @abstractmethod
+    def find_element(self, target: str, anchor: str, relation: str = "near", kind: str = "auto") -> "tuple[int, int]": ...
+
+    @abstractmethod
+    def tap_element(self, target: str, anchor: str, relation: str = "near", kind: str = "auto") -> None: ...
+
+    @abstractmethod
+    def input_text(self, locator: str, text: str) -> None: ...

automation/engines/vision_engine.py

@@ -0,0 +1,474 @@
+from __future__ import annotations
+
+import logging
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import numpy as np
+
+from automation import config
+from automation.engines.vision_base import VisionEngineBase
+from automation.utils import image_processing as ip, scroll_centering, spatial
+from automation.utils.cache import JsonFileCache
+from automation.utils.device_interface import DeviceInterface
+
+logger = logging.getLogger(__name__)
+
+# Vision-OCR-Cache (Methode: docs/recherche_caching.md). Inaktiv bei VISION_OCR_CACHE=False.
+_cache = JsonFileCache(config.VISION_CACHE_DIR)
+
+def _ocr(img) -> list:
+    """Full-Screen-OCR, in Zeilen gruppiert (teuer). Koordinaten stammen aus dem übergebenen Screenshot."""
+    return ip.ocr_group_lines(ip.ocr_find_text(img, lang=config.OCR_LANG))
+
+class VisionMatchError(RuntimeError):
+    def __init__(self, message: str, screenshot: "np.ndarray") -> None:
+        super().__init__(message)
+        self.screenshot = screenshot
+
+class VisionEngine(VisionEngineBase):
+
+    def __init__(self, device: DeviceInterface) -> None:
+        self._device = device
+
+    def _screen_key(self) -> tuple[str, str, str, str] | None:
+        """Stabile Screen-Identität (package, activity, version, device_fp) für den Cache.
+        None bei jedem Fehler -> Cache wird übersprungen, die Aktion läuft live weiter."""
+        return self._device.screen_key()
+
+    @staticmethod
+    def _padded_crop(img, box: dict, pad: int) -> tuple["np.ndarray", int, int] | None:
+        """Bbox um `pad` Pixel erweitert aus `img` croppen. Gibt (crop, x0, y0) oder None
+        bei leerem Crop; x0/y0 sind der Crop-Ursprung in Vollbild-Koordinaten."""
+        y0 = max(0, box["top"] - pad)
+        y1 = min(img.shape[0], box["top"] + box["height"] + pad)
+        x0 = max(0, box["left"] - pad)
+        x1 = min(img.shape[1], box["left"] + box["width"] + pad)
+        crop = img[y0:y1, x0:x1]
+        if crop.size == 0:
+            return None
+        return crop, x0, y0
+
+    @staticmethod
+    def _crop_ocr_match(img, box: dict, text: str) -> dict | None:
+        """Crop-OCR um eine gecachte Bbox auf dem Live-Screenshot. Gibt die Treffer-Zeile mit
+        Live-Koordinaten (in Vollbild-Koordinaten) zurück oder None."""
+        padded = VisionEngine._padded_crop(img, box, config.VISION_CROP_PAD)
+        if padded is None:
+            return None
+        crop, x0, y0 = padded
+        for line in ip.ocr_group_lines(ip.ocr_find_text(crop, lang=config.OCR_LANG)):
+            if text.lower() in line["text"].lower():
+                # Koordinaten zurück in Vollbild-Koordinaten verschieben (Crop-Ursprung addieren).
+                return {"text": line["text"],
+                        "left": line["left"] + x0, "top": line["top"] + y0,
+                        "width": line["width"], "height": line["height"]}
+        return None
+
+    def _locate_text(self, text: str, img) -> dict | None:
+        """Live-Bbox der ersten Zeile, die `text` enthält, sonst None. Cache an: Crop-OCR um die
+        gecachte Bbox; bei Miss Full-OCR-Fallback und Layout neu cachen."""
+        if not config.VISION_OCR_CACHE:
+            return next((h for h in _ocr(img) if text.lower() in h["text"].lower()), None)
+
+        key = self._screen_key()
+        entry = _cache.get(*key) if key is not None else None
+        cached = next((ln for ln in entry.get("elements", []) if text.lower() in ln["text"].lower()), None) if entry else None
+        live = self._crop_ocr_match(img, cached, text) if cached is not None else None
+        if live is not None:
+            logger.info("VISION_CACHE event=hit keyword=locate_text target=%r", text)
+            return live  # Treffer: Full-OCR übersprungen
+        if cached is not None:
+            logger.info("VISION_CACHE event=verify_fail keyword=locate_text target=%r", text)
+        else:
+            logger.info("VISION_CACHE event=miss keyword=locate_text target=%r", text)
+
+        # Miss oder Crop-Verifikation fehlgeschlagen: voller Live-Discovery-Fallback.
+        lines = _ocr(img)
+        if key is not None:
+            try:
+                _cache.put(key[0], key[1], key[2], key[3], elements=lines)
+            except Exception:
+                pass  # Cache-Schreibfehler darf die Aktion nie brechen
+        return next((h for h in lines if text.lower() in h["text"].lower()), None)
+
+    def text_exists(self, text: str, img: np.ndarray | None = None) -> bool:
+        if img is not None:
+            # Zwischen-Screenshot aus dem Scroll-Loop: nicht cachen (gleicher Screen-Key bei
+            # wechselndem Inhalt würde den Cache thrashen). Voll live lesen.
+            return any(text.lower() in h["text"].lower() for h in _ocr(img))
+        img = self._device.screenshot()
+        return self._locate_text(text, img) is not None
+
+    @staticmethod
+    def _screen_stable(img1: np.ndarray, img2: np.ndarray) -> bool:
+        import numpy as np
+        h = img1.shape[0]
+        crop1 = img1[h // 10 : 9 * h // 10]
+        crop2 = img2[h // 10 : 9 * h // 10]
+        return float(np.mean(np.abs(crop1.astype(float) - crop2.astype(float)))) < config.VISION_STABLE_THRESHOLD
+
+    def tap_text(self, text: str) -> None:
+        for attempt in (0, 1):
+            img = self._device.screenshot()
+            line = self._locate_text(text, img)
+            if line is not None:
+                cx = line["left"] + line["width"] // 2
+                cy = line["top"] + line["height"] // 2
+                self._device.tap(cx, cy)
+                return
+            if attempt == 0:
+                # Text nicht gefunden, kurze Pause damit laufende Animationen abklingen können.
+                time.sleep(1.0)
+        raise VisionMatchError(f"Text not found: '{text}'", screenshot=img)
+
+    def _resolve_target_hit(self, hits: list[dict], text: str, anchor: str | None, relation: str) -> dict | None:
+        """Treffer für `text` unter den OCR-Zeilen `hits`. Ohne anchor der erste Treffer, mit
+        anchor der Treffer, der `relation` zum Anker erfüllt."""
+        if anchor is None:
+            return next((h for h in hits if text.lower() in h["text"].lower()), None)
+        anchor_box = next((h for h in hits if anchor.lower() in h["text"].lower()), None)
+        if anchor_box is None:
+            return None  # Anker in diesem Frame nicht sichtbar -> keine Aussage, kein False-Positive
+        candidates = [h for h in hits if text.lower() in h["text"].lower() and h is not anchor_box]
+        return spatial.resolve(
+            anchor_box, candidates, relation,
+            overlap_tol=config.SPATIAL_OVERLAP_TOLERANCE,
+            max_distance=config.SPATIAL_NEAR_MAX_DISTANCE,
+        )
+
+    def _scroll_and_search(self, x: int, center: int, step: int, text: str, direction: int = 1,
+                            anchor: str | None = None, relation: str = "near") -> bool:
+        prev_img: np.ndarray | None = None
+        for i in range(config.VISION_SCROLL_MAX):
+            self._device.scroll(
+                x, center, step if direction == 1 else -step,
+                duration_ms=config.VISION_SCROLL_SWIPE_DURATION_MS,
+            )
+            img = self._device.screenshot()
+            hits = _ocr(img)
+
+            hit = self._resolve_target_hit(hits, text, anchor, relation)
+            logger.debug("scroll_and_search iteration=%d direction=%d hit=%s", i, direction, hit is not None)
+            if hit is not None:
+                logger.info("scroll_and_search event=found target=%r direction=%d iteration=%d", text, direction, i)
+                height = center * 2
+                el_center_y = hit["top"] + hit["height"] // 2
+                scroll_centering.center_in_safe_area(self._device, x, height, el_center_y, max_shift=step)
+                return True
+
+            if anchor is not None:
+                # Anker re-lokalisieren, deckt fixe und mitscrollende Anker gleichermaßen ab.
+                # Kein Treffer -> x, center bleiben unverändert, kein Rückfall auf Bildschirmmitte.
+                anchor_box = next((h for h in hits if anchor.lower() in h["text"].lower()), None)
+                if anchor_box is not None:
+                    x, center = spatial.center(anchor_box)
+
+            if prev_img is not None and self._screen_stable(prev_img, img):
+                logger.info("scroll_and_search event=screen_stable target=%r direction=%d iteration=%d", text, direction, i)
+                break
+            prev_img = img
+        else:
+            logger.info("scroll_and_search event=exhausted target=%r direction=%d max=%d", text, direction, config.VISION_SCROLL_MAX)
+        return False
+
+    @staticmethod
+    def _resolve_type(target: str, kind: str) -> str:
+        # "auto": .png-Pfad => Icon (Template-Match), sonst Text (OCR).
+        if kind != "auto":
+            return kind
+        return "icon" if target.lower().endswith(".png") else "text"
+
+    @staticmethod
+    def _verify_cached_boxes(img, custom_element, cached: list[dict], ew: int, eh: int) -> list[dict]:
+        """Live-Crop-Verifikation gecachter Custom-Element-Boxen. Boxen ohne Live-Treffer fallen raus."""
+        verified = []
+        for box in cached:
+            padded = VisionEngine._padded_crop(img, box, config.VISION_CROP_PAD)
+            if padded is None:
+                continue
+            crop, x0, y0 = padded
+            hits = ip.match_custom_element(crop, custom_element, threshold=config.MATCH_THRESHOLD)
+            if not hits:
+                continue
+            cx_crop, cy_crop = hits[0]
+            # Koordinaten zurück in Vollbild-Koordinaten verschieben (Crop-Ursprung addieren).
+            verified.append({
+                "left": x0 + cx_crop - ew // 2,
+                "top":  y0 + cy_crop - eh // 2,
+                "width": ew, "height": eh,
+            })
+        return verified
+
+    def _custom_element_cached(self, img, element_path: str) -> list[dict]:
+        """Custom-Element-Matches als Boxes. Wie _locate_text, aber Template-Matching statt OCR:
+        Cache an -> Crop-Match um gecachte Boxen, bei Miss Full-Screen-Fallback und neu cachen."""
+        custom_element = ip.load_image(element_path)
+        eh, ew = custom_element.shape[:2]
+
+        def _run_full() -> list[dict]:
+            matches = ip.filter_matches(
+                ip.match_custom_element(img, custom_element, threshold=config.MATCH_THRESHOLD)
+            )
+            # match_custom_element liefert Mittelpunkte; daraus eine Box in Bildkoordinaten bauen.
+            return [{"left": cx - ew // 2, "top": cy - eh // 2, "width": ew, "height": eh}
+                    for cx, cy in matches]
+
+        if not config.VISION_OCR_CACHE:
+            return _run_full()
+
+        key = self._screen_key()
+        cached = _cache.get_custom_element_matches(*key, element_path) if key is not None else None
+        verified = self._verify_cached_boxes(img, custom_element, cached, ew, eh) if cached is not None else []
+        if verified:
+            logger.info("VISION_CACHE event=hit keyword=custom_element_cached element_path=%r boxes=%d",
+                        element_path, len(verified))
+            return verified  # Treffer: Full-Screen-Match übersprungen
+        if cached is not None:
+            logger.info("VISION_CACHE event=verify_fail keyword=custom_element_cached element_path=%r boxes=%d",
+                        element_path, len(cached))
+        else:
+            logger.info("VISION_CACHE event=miss keyword=custom_element_cached element_path=%r", element_path)
+
+        # Miss oder Crop-Verifikation fehlgeschlagen: voller Live-Discovery-Fallback.
+        boxes = _run_full()
+        if key is not None:
+            try:
+                _cache.put_custom_element_matches(*key, element_path, boxes)
+            except Exception:
+                pass  # Cache-Schreibfehler darf die Aktion nie brechen
+        return boxes
+
+    def _find_candidates(self, img, lines: list, target: str, kind: str) -> list[dict]:
+        """Kandidaten-Boxes für das Ziel: OCR-Zeilen (Text) oder Template-Matches (Icon)."""
+        if kind == "icon":
+            return self._custom_element_cached(img, target)
+        return [line for line in lines if target.lower() in line["text"].lower()]
+
+    def _locate_anchor_cached(self, img, anchor: str) -> tuple[dict, dict, dict] | None:
+        """Cache-Lookup + Live-Crop-Verify für den Anker-Text. Gibt (live_anchor, cached_anchor,
+        entry) zurück oder None bei Cache-Miss."""
+        if not config.VISION_OCR_CACHE:
+            return None
+        key = self._screen_key()
+        if key is None:
+            return None
+        entry = _cache.get(*key)
+        if not entry:
+            return None
+        cached_anchor = next(
+            (ln for ln in entry.get("elements", []) if anchor.lower() in ln["text"].lower()), None
+        )
+        if cached_anchor is None:
+            return None
+        live_anchor = self._crop_ocr_match(img, cached_anchor, anchor)
+        if live_anchor is None:
+            return None
+        return live_anchor, cached_anchor, entry
+
+    def _locate_relation(self, img, anchor: str, target: str) -> tuple[dict, list[dict]] | None:
+        """Cache-gestützter Lookup für Anker + Text-Ziel-Kandidaten, alle Boxen live crop-verifiziert.
+        None bei Cache-aus/Miss -> find_element fällt dann auf Full-OCR zurück."""
+        located = self._locate_anchor_cached(img, anchor)
+        if located is None:
+            return None
+        live_anchor, cached_anchor, entry = located
+        live_candidates = []
+        for ln in entry.get("elements", []):
+            # Anker nicht als eigenes Ziel: Identität auf den Cache-Zeilen (Analogie zu
+            # `c is not anchor_box` im Full-OCR-Pfad). Merge-Fall -> kein Kandidat -> Full-OCR-Fallback.
+            if ln is cached_anchor:
+                continue
+            if target.lower() in ln["text"].lower():
+                live = self._crop_ocr_match(img, ln, target)
+                if live is not None:
+                    live_candidates.append(live)
+        if not live_candidates:
+            return None
+        return live_anchor, live_candidates
+
+    def find_element(self, target: str, anchor: str, relation: str = "near", kind: str = "auto") -> tuple[int, int]:
+        """Lokalisiert das Ziel live relativ zum Anker-Text und gibt den Mittelpunkt (x, y)
+        zurück. Der Cache dient nur als Kandidatenquelle, Koordinaten stammen immer live."""
+        for attempt in (0, 1):
+            try:
+                return self._find_element_once(target, anchor, relation, kind)
+            except RuntimeError:
+                if attempt == 0:
+                    # Kein Treffer - kurze Pause für den Fall, dass Inhalte (z. B. WebView2-Content
+                    # nach einer Navigation) noch nicht fertig gerendert sind.
+                    time.sleep(1.0)
+                    continue
+                raise
+
+    def _cached_anchor_and_candidates(self, img, target: str, anchor: str, kind: str) -> tuple[dict, list[dict]] | None:
+        """Cache-Pfad für Anker + Kandidaten, ohne Full-OCR. None wenn der Cache keinen
+        brauchbaren Treffer liefert, dann fällt _find_element_once auf Full-OCR zurück."""
+        if kind == "text":
+            return self._locate_relation(img, anchor, target)
+        # kind == "icon": Anker per Cache (kein Full-OCR), Icon-Kandidaten per _custom_element_cached
+        # (hat selbst einen Full-Screen-Fallback, falls das Template warm nicht matcht).
+        located = self._locate_anchor_cached(img, anchor)
+        if located is None:
+            return None
+        return located[0], self._find_candidates(img, [], target, "icon")
+
+    def _full_ocr_anchor_and_candidates(self, img, target: str, anchor: str, kind: str) -> tuple[dict, list[dict]]:
+        """Full-OCR-Fallback: sucht Anker und Kandidaten live im gesamten Frame, cached das
+        Zeilen-Layout fürs nächste Mal (Anker ist immer Text)."""
+        lines = _ocr(img)
+        if config.VISION_OCR_CACHE:
+            key = self._screen_key()
+            if key is not None:
+                try:
+                    _cache.put(key[0], key[1], key[2], key[3], elements=lines)
+                except Exception:
+                    pass
+        anchor_box = next((line for line in lines if anchor.lower() in line["text"].lower()), None)
+        if anchor_box is None:
+            raise VisionMatchError(f"Anchor text not found: '{anchor}'", screenshot=img)
+        candidates = self._find_candidates(img, lines, target, kind)
+        candidates = [c for c in candidates if c is not anchor_box]  # Anker nicht als eigenes Ziel
+        return anchor_box, candidates
+
+    def _find_element_once(self, target: str, anchor: str, relation: str, kind: str) -> tuple[int, int]:
+        kind = self._resolve_type(target, kind)
+        img = self._device.screenshot()
+
+        cached = self._cached_anchor_and_candidates(img, target, anchor, kind)
+        logger.info("VISION_CACHE event=%s keyword=find_element target=%r anchor=%r kind=%s",
+                    "hit" if cached is not None else "miss", target, anchor, kind)
+        anchor_box, candidates = (
+            cached if cached is not None else self._full_ocr_anchor_and_candidates(img, target, anchor, kind)
+        )
+
+        if not candidates:
+            raise VisionMatchError(f"Target not found: '{target}'", screenshot=img)
+        best = spatial.resolve(
+            anchor_box, candidates, relation,
+            overlap_tol=config.SPATIAL_OVERLAP_TOLERANCE,
+            max_distance=config.SPATIAL_NEAR_MAX_DISTANCE,
+        )
+        if best is None:
+            raise VisionMatchError(f"No '{target}' {relation} '{anchor}'", screenshot=img)
+        return spatial.center(best)
+
+    def tap_element(self, target: str, anchor: str, relation: str = "near", kind: str = "auto") -> None:
+        # Re-lokalisiert live und tippt sofort, kein Tippen auf zwischengespeicherte Koordinaten.
+        x, y = self.find_element(target, anchor, relation=relation, kind=kind)
+        self._device.tap(x, y)
+
+    def _disambiguate(self, boxes: list[dict], locator: str, region: tuple[int, int, int, int] | None,
+                       img) -> dict:
+        """Wählt bei mehreren Treffern den einen Treffer innerhalb von region. Ohne
+        bestimmbaren Bereich wird kontrolliert abgebrochen statt zu raten."""
+        if region is None:
+            raise VisionMatchError(
+                f"Mehrdeutig: {len(boxes)} Treffer für '{locator}', kein aktiver Dialog- oder "
+                "Feldbereich erkennbar. Eindeutigen Locator verwenden.",
+                screenshot=img,
+            )
+        left, top, right, bottom = region
+        in_region = [b for b in boxes if left <= spatial.center(b)[0] <= right
+                                      and top <= spatial.center(b)[1] <= bottom]
+        if len(in_region) != 1:
+            raise VisionMatchError(
+                f"Mehrdeutig: {len(in_region)} von {len(boxes)} Treffern für '{locator}' im "
+                "aktiven Dialogbereich. Eindeutigen Locator verwenden.",
+                screenshot=img,
+            )
+        return in_region[0]
+
+    def input_text(self, locator: str, text: str) -> None:
+        """Lokalisiert das Feld per locator, tippt den Mittelpunkt, verifiziert den Fokus im
+        Zielprozess und gibt dann literalen Text ein. Ohne verifizierten Fokus wird nicht
+        geschrieben."""
+        kind = self._resolve_type(locator, "auto")
+        img = self._device.screenshot()
+
+        try:
+            region = self._device.active_region()
+            supports_active_region = True
+        except NotImplementedError:
+            region = None
+            supports_active_region = False
+
+        if not supports_active_region:
+            if kind == "icon":
+                boxes = self._custom_element_cached(img, locator)
+                if not boxes:
+                    raise VisionMatchError(f"Eingabefeld nicht gefunden (Custom Element): '{locator}'", screenshot=img)
+                box = boxes[0]
+            else:
+                box = self._locate_text(locator, img)
+                if box is None:
+                    raise VisionMatchError(f"Eingabefeld nicht gefunden (Text): '{locator}'", screenshot=img)
+        else:
+            if kind == "icon":
+                boxes = self._custom_element_cached(img, locator)
+            else:
+                boxes = [h for h in _ocr(img) if locator.lower() in h["text"].lower()]
+            if not boxes:
+                raise VisionMatchError(f"Eingabefeld nicht gefunden ({kind}): '{locator}'", screenshot=img)
+            if len(boxes) == 1:
+                box = boxes[0]
+            else:
+                if region is None and kind == "text":
+                    # Zweites Signal für Apps ohne eigenes Overlay-Fenster.
+                    try:
+                        region = self._device.value_region(locator)
+                    except NotImplementedError:
+                        region = None
+                    logger.info(
+                        "input_text event=value_region_lookup locator=%r result=%s",
+                        locator, "hit" if region is not None else "miss",
+                    )
+                box = self._disambiguate(boxes, locator, region, img)
+
+        cx = box["left"] + box["width"] // 2
+        cy = box["top"] + box["height"] // 2
+        self._device.tap(cx, cy)
+        time.sleep(0.3)
+        if not self._device.verify_focus_in_target():
+            raise VisionMatchError(
+                f"Fokus nach Click auf '{locator}' liegt nicht im verbundenen Zielprozess, "
+                "Eingabe abgebrochen.",
+                screenshot=img,
+            )
+        try:
+            self._device.clear_focused_field(cx, cy)
+        except NotImplementedError:
+            pass
+        except Exception as exc:
+            raise VisionMatchError(
+                f"Feldinhalt vor Eingabe in '{locator}' konnte nicht gelöscht oder verifiziert "
+                f"werden: {exc}",
+                screenshot=img,
+            ) from exc
+        self._device.type_text(text)
+
+    def tap_icon_near_text(self, icon_path: str, text: str) -> None:
+        # Rückwärtskompatibel: "Icon neben Text" ist der near-Spezialfall der Relations-API.
+        self.tap_element(icon_path, anchor=text, relation="near", kind="icon")
+
+    def scroll_to_text(self, text: str, anchor: str | None = None, relation: str = "near") -> bool:
+        img = self._device.screenshot()
+        height, width = img.shape[0], img.shape[1]
+        step = int(height * config.VISION_SCROLL_STEP_PERCENT)
+        center = height // 2
+        x = width // 2
+        hits = _ocr(img)
+
+        if anchor is not None:
+            anchor_box = next((h for h in hits if anchor.lower() in h["text"].lower()), None)
+            if anchor_box is not None:
+                x, center = spatial.center(anchor_box)
+            # Anker im Startbild nicht sichtbar: Fallback auf Bildschirmmitte, kein Hard-Error
+            # (Anker könnte erst nach Scrollen erscheinen)
+
+        if self._resolve_target_hit(hits, text, anchor, relation) is not None:
+            return True
+        if self._scroll_and_search(x, center, step, text, direction=1, anchor=anchor, relation=relation):
+            return True
+        return self._scroll_and_search(x, center, step, text, direction=-1, anchor=anchor, relation=relation)