fusiontest 0.2.4__tar.gz → 0.2.6__tar.gz

{fusiontest-0.2.4 → fusiontest-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.4
+Version: 0.2.6
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.4 → fusiontest-0.2.6}/fusiontest/core/action_model.py

@@ -87,6 +87,23 @@ def _is_backend_specific_error(err: Exception) -> bool:
     return any(tok in msg for tok in _BACKEND_SPECIFIC_TOKENS)
 
 
+# A single request bigger than this backend accepts: context window, or a
+# per-minute token allowance smaller than the request itself (Groq free tier:
+# 8,000 TPM; a goal-discovery prompt can be ~9k). Waiting won't help, but the
+# backend is fine for smaller requests — so fall through without a cooldown.
+_TOO_LARGE_TOKENS = (
+    "request too large", "reduce your message size", "context_length_exceeded",
+    "maximum context length", "prompt is too long", "input token count",
+)
+
+
+def _is_request_too_large(err: Exception) -> bool:
+    if _error_status_code(err) == 413:
+        return True
+    msg = str(err).lower()
+    return any(tok in msg for tok in _TOO_LARGE_TOKENS)
+
+
 def _is_rate_limit(err: Exception) -> bool:
     if err.__class__.__name__ == "RateLimitError":
         return True
@@ -536,6 +553,8 @@ class LLMCaller:
             then fall through.
           - On backend-specific errors (retired model, bad/missing key for
             that provider): start a 5-minute cooldown and fall through.
+          - On a request too large for that backend (413, context length):
+            fall through without a cooldown — smaller requests still fit.
           - On other non-transient errors (bad request): raise immediately.
 
         Raises LLMUnavailableError when no entry could answer.
@@ -561,6 +580,12 @@ class LLMCaller:
                     return answer
                 except Exception as err:  # noqa: BLE001
                     last_error = err
+                    if _is_request_too_large(err):
+                        logger.warning(
+                            "%s:%s can't take a request this size (%s) — falling through.",
+                            backend, model or "auto", err.__class__.__name__,
+                        )
+                        break
                     transient = _is_transient_error(err)
                     rate_limited = _is_rate_limit(err)
 

{fusiontest-0.2.4 → fusiontest-0.2.6}/fusiontest/core/runner.py

@@ -179,6 +179,10 @@ class RunnerConfig:
     # are replayed without LLM calls; the LLM re-verifies every N replays.
     trace_store: TraceStore | None = None
     reverify_every: int = 10
+    # Stop a goal once the screen hasn't changed for this many consecutive
+    # steps (actions with no visible effect, or no usable action) instead of
+    # spending LLM calls until max_steps_per_goal.
+    max_stalled_steps: int = 3
 
 
 # ── Runner ─────────────────────────────────────────────────────────────────────
@@ -437,6 +441,9 @@ class FusionTestRunner:
 
         self._navigate(navigate_url)
 
+        prev_screen: str | None = None   # screen at the previous step
+        stalled = 0                      # consecutive steps with an unchanged screen
+
         for step_num in range(1, self.config.max_steps_per_goal + 1):
             step.steps_taken = step_num
 
@@ -454,6 +461,23 @@ class FusionTestRunner:
             screen_text = _redact(screen_text)
             _drop_if_no_effect(screen_text)
 
+            # No progress: tell the model its last action did nothing, and stop
+            # the goal once nothing has changed for max_stalled_steps steps.
+            if prev_screen is not None and screen_text == prev_screen:
+                stalled += 1
+                if history and not history[-1].endswith(_NO_EFFECT_NOTE):
+                    history[-1] += _NO_EFFECT_NOTE
+            else:
+                stalled = 0
+            prev_screen = screen_text
+            if stalled >= self.config.max_stalled_steps:
+                logger.info(f"  No progress for {stalled} steps — stopping this goal")
+                self._last_chance_verify(
+                    step, goal, screen_text, history, recorded,
+                    reason=f"no_progress: the screen didn't change for {stalled} consecutive steps",
+                )
+                break
+
             # NOTE: We record the screen hash AFTER the model decides, not
             # before. This prevents observation/verification goals from
             # triggering screen loop detection when the page doesn't change.
@@ -609,16 +633,10 @@ class FusionTestRunner:
             except Exception:
                 final_screen_text = screen_text  # fall back to last known screen
 
-            step.error = f"max_steps_exceeded ({self.config.max_steps_per_goal})"
-            if self.verifier.is_complete(goal, final_screen_text, history):
-                step.success = True
-                step.error = ""
-                self._last_recording = self._make_recording(recorded, final_screen_text)
-                logger.info(
-                    f"  Goal verified complete at max_steps "
-                    f"({self.config.max_steps_per_goal}) — last-chance check passed"
-                )
-            self._add_verifier_tokens(step)
+            self._last_chance_verify(
+                step, goal, final_screen_text, history, recorded,
+                reason=f"max_steps_exceeded ({self.config.max_steps_per_goal})",
+            )
 
         if not step.success and self.config.screenshot_on_failure:
             try:
@@ -755,6 +773,26 @@ class FusionTestRunner:
                 )
         return replace(action, value=resolver.resolve(action.value)), None
 
+    def _last_chance_verify(
+        self, step: StepResult, goal: str, screen_text: str, history: list[str],
+        recorded: list[RecordedAction], reason: str,
+    ) -> None:
+        """The goal is ending without the model saying "done" (step limit or no
+        progress): ask the verifier once whether it's met anyway. Otherwise the
+        goal fails with `reason` — or errors, if the verifier couldn't answer."""
+        step.error = reason
+        if self.verifier.is_complete(goal, screen_text, history):
+            step.success = True
+            step.error = ""
+            self._last_recording = self._make_recording(recorded, screen_text)
+            logger.info(f"  Goal verified complete on last-chance check ({reason})")
+        else:
+            failure = getattr(self.verifier, "last_failure_detail", None)
+            if isinstance(failure, str):
+                step.infra_error = True
+                step.error = f"verifier_unavailable: verifier: {failure}"[:2000]
+        self._add_verifier_tokens(step)
+
     def _add_verifier_tokens(self, step: StepResult) -> None:
         usage = getattr(self.verifier, "last_token_usage", None)
         if isinstance(usage, TokenUsage):
@@ -789,6 +827,9 @@ class FusionTestRunner:
 
 # ── Secret binding helpers ─────────────────────────────────────────────────────
 
+# Appended to the model's history entry for an action that changed nothing.
+_NO_EFFECT_NOTE = " → no visible change on screen"
+
 # Recorded actions that are dropped from a trace when they didn't change the screen.
 _PRUNABLE_ACTIONS = {"tap", "scroll", "swipe", "wait"}
 

{fusiontest-0.2.4 → fusiontest-0.2.6}/fusiontest/desktop/playwright_adapter.py

@@ -35,6 +35,33 @@ _ARIA_BARE_RE = re.compile(r'^(\w+)\s*(?:\[[^\]]*\]\s*)*:?\s*$')
 _MAX_WAIT_SECONDS = 10.0
 
 
+# Invalid form fields after user interaction / a submit attempt (see
+# PlaywrightAdapter._validation_messages).
+_VALIDATION_JS = """() => Array.from(document.querySelectorAll('input, textarea, select'))
+  .filter(el => { try { return el.matches(':user-invalid'); } catch (e) { return false; } })
+  .map(el => ({
+    label: ((el.labels && el.labels[0] && el.labels[0].innerText) || el.getAttribute('aria-label')
+            || el.placeholder || el.name || el.type || 'field').trim().slice(0, 80),
+    message: (el.validationMessage || '').trim().slice(0, 160),
+  }))
+  .filter(f => f.message)
+  .slice(0, 20)"""
+
+
+def _with_validation_section(screen_text: str, messages: list[tuple[str, str]]) -> str:
+    """Insert a "-- Form validation --" section before the page content (so the
+    per-step action prompt, which trims page content, keeps it)."""
+    if not messages:
+        return screen_text
+    section = "-- Form validation --\n" + "\n".join(
+        f'  field "{label}": {message}' for label, message in messages
+    )
+    marker = "-- Page content --"
+    if marker in screen_text:
+        return screen_text.replace(marker, f"{section}\n{marker}", 1)
+    return f"{screen_text.rstrip()}\n{section}"
+
+
 class AdapterAborted(RuntimeError):
     """The adapter was aborted (e.g. by the per-goal watchdog) mid-call."""
 
@@ -278,12 +305,28 @@ class PlaywrightAdapter:
                 elements = dom_elements
                 screen_text = dom_text
 
+        screen_text = _with_validation_section(screen_text, await self._validation_messages())
+
         # Prepend page context so the LLM knows where it is
         header = f"=== PAGE: {page_title} ===\nURL: {page_url}\n"
         screen_text = header + screen_text
 
         return elements, screen_text
 
+    async def _validation_messages(self) -> list[tuple[str, str]]:
+        """(field label, message) for form fields the browser flags as invalid.
+
+        Native validation bubbles ("Please fill out this field.") aren't in the
+        accessibility tree, so without this the model and verifier can't see
+        them. Only :user-invalid fields are reported — ones the user interacted
+        with or tried to submit — so untouched required fields aren't noise.
+        """
+        try:
+            return [(f["label"], f["message"]) for f in await self._page.evaluate(_VALIDATION_JS)]
+        except Exception as exc:  # noqa: BLE001 — best effort; never break screen reads
+            logger.debug("Could not read form validation state: %s", exc)
+            return []
+
     async def _dom_fallback_screen(self) -> tuple[list[UIElement], str]:
         """Extract interactive elements directly from the DOM via JavaScript.
         This catches elements that aria_snapshot misses on JS-heavy pages."""

{fusiontest-0.2.4 → fusiontest-0.2.6}/fusiontest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.4
+Version: 0.2.6
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.4 → fusiontest-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fusiontest"
-version = "0.2.4"
+version = "0.2.6"
 description = "AI-powered UI testing for mobile and desktop — by FusionLeap.io"
 readme = "README.md"
 requires-python = ">=3.10"