fusiontest 0.2.2__tar.gz → 0.2.4__tar.gz

{fusiontest-0.2.2 → fusiontest-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.2
+Version: 0.2.4
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest/core/action_model.py

@@ -209,6 +209,57 @@ GEMINI_DEFAULT_MODEL = "gemini-2.5-flash"
 # yields empty/truncated output. Billing is per token used, not the cap.
 LLM_MAX_OUTPUT_TOKENS = 1024
 
+# Per-request timeout for every provider client. Without one, a hung request
+# blocks a goal until the per-goal watchdog; a timeout is transient, so the
+# chain retries or falls through to the next backend.
+LLM_REQUEST_TIMEOUT_SECONDS = 120
+
+
+# Page text the action model sees per step (ADR-007 cost work). It needs every
+# clickable element but only the gist of the page — the verifier, which judges
+# content, gets the full page. Measured on fusionleap.io, page text was ~65% of
+# each ~2.6k-token step prompt.
+_ACTION_CONTENT_CHARS = 1500
+_ACTION_LABEL_CHARS = 80
+_ACTION_ELEMENT_CHARS = 120
+_CONTENT_LINE_RE = re.compile(r'^\s*(\w+): "(.*)"$')
+_ELEMENT_LINE_RE = re.compile(r'^\s*\[\d+\] ')
+
+
+def compact_screen_for_action(screen_text: str) -> str:
+    """Trim a screen's page-content section for the per-step action prompt.
+
+    Every interactive element is kept (the model can only act on what it sees),
+    but very long labels — e.g. list items holding whole paragraphs — are cut:
+    the model acts by index. In the page content, headings are kept whole,
+    other text is cut to 80 characters, and the section stops at ~1,500 characters.
+    """
+    lines: list[str] = []
+    in_content = False
+    used = 0
+    for line in screen_text.splitlines():
+        if line.startswith("-- "):
+            in_content = line.startswith("-- Page content")
+            lines.append(line)
+            continue
+        if not in_content:
+            if len(line) > _ACTION_ELEMENT_CHARS and _ELEMENT_LINE_RE.match(line):
+                line = line[:_ACTION_ELEMENT_CHARS] + '…"'
+            lines.append(line)
+            continue
+        match = _CONTENT_LINE_RE.match(line)
+        if match:
+            kind, label = match.groups()
+            if kind != "heading" and len(label) > _ACTION_LABEL_CHARS:
+                label = label[:_ACTION_LABEL_CHARS] + "…"
+            line = f'  {kind}: "{label}"'
+        if used + len(line) > _ACTION_CONTENT_CHARS:
+            lines.append("  … (more page text omitted for brevity)")
+            break
+        used += len(line)
+        lines.append(line)
+    return "\n".join(lines)
+
 
 def openai_compat_extra_args(model: str) -> dict:
     """Extra chat.completions args for OpenAI-compatible models (Groq/OpenAI/Ollama)."""
@@ -321,6 +372,10 @@ class LLMRequest:
     json_object: bool = False   # OpenAI-compatible response_format={"type": "json_object"}
     empty: str = ""             # returned when the model produces no text
     max_output_tokens: int = LLM_MAX_OUTPUT_TOKENS  # includes hidden reasoning/thinking
+    # Let reasoning models think before answering. Off for per-step action and
+    # verifier calls (short answers; thinking tokens are billed as output and
+    # add latency); on for open-ended generation like goal discovery.
+    thinking: bool = False
 
 
 class LLMCaller:
@@ -399,7 +454,7 @@ class LLMCaller:
                     "OPENAI_API_KEY is not set. Add it to your .env file or run: "
                     "export OPENAI_API_KEY=sk-..."
                 )
-            client = OpenAI(api_key=api_key)
+            client = OpenAI(api_key=api_key, timeout=LLM_REQUEST_TIMEOUT_SECONDS)
 
         elif backend == "claude":
             try:
@@ -412,7 +467,7 @@ class LLMCaller:
                     "ANTHROPIC_API_KEY is not set. Add it to your .env file or run: "
                     "export ANTHROPIC_API_KEY=sk-ant-..."
                 )
-            client = anthropic.Anthropic(api_key=api_key)
+            client = anthropic.Anthropic(api_key=api_key, timeout=LLM_REQUEST_TIMEOUT_SECONDS)
 
         elif backend == "ollama":
             try:
@@ -422,7 +477,7 @@ class LLMCaller:
                     "Ollama backend requires the openai SDK: pip install openai"
                 ) from err
             base_url = os.getenv("OLLAMA_HOST", "http://localhost:11434") + "/v1"
-            client = OpenAI(base_url=base_url, api_key="ollama")
+            client = OpenAI(base_url=base_url, api_key="ollama", timeout=LLM_REQUEST_TIMEOUT_SECONDS)
 
         elif backend == "groq":
             try:
@@ -435,7 +490,8 @@ class LLMCaller:
                     "GROQ_API_KEY is not set. Get a free key at https://console.groq.com "
                     "then: export GROQ_API_KEY=gsk_..."
                 )
-            client = OpenAI(base_url="https://api.groq.com/openai/v1", api_key=api_key)
+            client = OpenAI(base_url="https://api.groq.com/openai/v1", api_key=api_key,
+                            timeout=LLM_REQUEST_TIMEOUT_SECONDS)
 
         elif backend == "gemini":
             try:
@@ -452,7 +508,11 @@ class LLMCaller:
                     "GEMINI_API_KEY is not set. Add it to your .env file or run: "
                     "export GEMINI_API_KEY=AIza..."
                 )
-            client = genai.Client(api_key=api_key)
+            from google.genai import types as genai_types
+            client = genai.Client(
+                api_key=api_key,
+                http_options=genai_types.HttpOptions(timeout=LLM_REQUEST_TIMEOUT_SECONDS * 1000),  # ms
+            )
 
         else:
             raise RuntimeError(f"Unknown backend: {backend}")
@@ -609,6 +669,11 @@ class LLMCaller:
                     system_instruction=request.system,
                     temperature=self.temperature,
                     max_output_tokens=request.max_output_tokens,
+                    # Flash models can skip thinking entirely; Pro models can't.
+                    thinking_config=(
+                        genai_types.ThinkingConfig(thinking_budget=0)
+                        if not request.thinking and "flash" in resolved_model else None
+                    ),
                 ),
             )
             meta = getattr(resp, "usage_metadata", None)
@@ -820,7 +885,7 @@ class ActionModel(LLMCaller):
             pruned_invalid = invalid_actions[-self.max_invalid_actions_in_prompt:]
             sections += ["", "ACTIONS THAT FAILED (do NOT repeat):", *[f"  - {a}" for a in pruned_invalid]]
 
-        trimmed_screen = screen_text
+        trimmed_screen = compact_screen_for_action(screen_text)
         if self.max_screen_chars and len(trimmed_screen) > self.max_screen_chars:
             # Truncate at the last newline before the limit to avoid cutting an
             # element in half. Interactive elements are listed first, so only

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest/core/runner.py

@@ -60,7 +60,8 @@ class StepResult:
     screenshots: list[bytes] = field(default_factory=list)
     error: str = ""
     duration_seconds: float = 0.0
-    token_usage: TokenUsage = field(default_factory=TokenUsage)
+    token_usage: TokenUsage = field(default_factory=TokenUsage)       # all LLM calls
+    verifier_tokens: TokenUsage = field(default_factory=TokenUsage)   # the verifier's share
     # The goal could not be evaluated (LLM capacity / infrastructure). Not a
     # test result: excluded from stability and reported as an error (ADR-007).
     infra_error: bool = False
@@ -80,6 +81,10 @@ class StepResult:
             "error":            self.error,
             "duration_seconds": self.duration_seconds,
             "token_usage":      self.token_usage.to_dict(),
+            "token_breakdown":  {
+                "action":   (self.token_usage - self.verifier_tokens).to_dict(),
+                "verifier": self.verifier_tokens.to_dict(),
+            },
             "mode":             self.mode,
         }
 
@@ -313,12 +318,15 @@ class FusionTestRunner:
                         goal_num, total_goals, limit,
                     )
                     fired.set()
-                    if cancel_event is not None:
-                        cancel_event.set()
+                    # Runs on the timer thread: use the adapter's thread-safe
+                    # abort so the in-flight call fails now, not when it returns.
+                    # (Not cancel_event — that's the user's Stop button.)
+                    abort = getattr(self.adapter, "abort", None) or getattr(self.adapter, "stop", None)
                     try:
-                        self.adapter.stop()
+                        if abort is not None:
+                            abort()
                     except Exception as stop_exc:  # noqa: BLE001
-                        logger.debug("Adapter stop during watchdog raised: %s", stop_exc)
+                        logger.warning("Adapter abort during watchdog raised: %s", stop_exc)
 
                 watchdog = threading.Timer(self.config.max_goal_seconds, _on_timeout)
                 watchdog.daemon = True
@@ -326,18 +334,21 @@ class FusionTestRunner:
 
             try:
                 step_result = self._execute_goal(goal, safe_goal, goal_url, resolver)
+            except Exception:
+                # The watchdog closing the browser makes the in-flight call raise.
+                if not watchdog_fired.is_set():
+                    raise
+                step_result = StepResult(goal=safe_goal, success=False, steps_taken=0)
             finally:
                 if watchdog is not None:
                     watchdog.cancel()
 
-            # If the watchdog actually fired, record a clear timeout reason on
-            # the step so the UI can show "goal_timeout" rather than whatever
-            # generic Playwright error bubbled up. Also force-abort the rest of
-            # the run since the adapter has been stopped and all subsequent
-            # goals would fail instantly anyway.
+            # A timed-out goal couldn't be evaluated: record it as an error
+            # (not a test failure), and stop the run — the browser is closed.
             goal_timed_out = watchdog_fired.is_set()
             if goal_timed_out:
                 step_result.success = False
+                step_result.infra_error = True
                 step_result.error = (
                     f"goal_timeout: exceeded {self.config.max_goal_seconds}s wall-clock limit"
                 )
@@ -367,7 +378,7 @@ class FusionTestRunner:
 
             # Watchdog closed the adapter — no point attempting further goals.
             if goal_timed_out:
-                result.cancelled = True
+                result.error = step_result.error
                 result.success = False
                 logger.error(
                     "Aborting remaining %d goal(s) after per-goal timeout",
@@ -506,7 +517,7 @@ class FusionTestRunner:
                     action_str = _redact(f"done: {action.reasoning}")
                     step.actions.append(action_str)
                     verified = self.verifier.is_complete(goal, screen_text, history)
-                    step.token_usage += getattr(self.verifier, "last_token_usage", TokenUsage())
+                    self._add_verifier_tokens(step)
                     step.success = verified
                     if verified:
                         self._last_recording = self._make_recording(recorded, screen_text)
@@ -607,7 +618,7 @@ class FusionTestRunner:
                     f"  Goal verified complete at max_steps "
                     f"({self.config.max_steps_per_goal}) — last-chance check passed"
                 )
-            step.token_usage += getattr(self.verifier, "last_token_usage", TokenUsage())
+            self._add_verifier_tokens(step)
 
         if not step.success and self.config.screenshot_on_failure:
             try:
@@ -744,6 +755,12 @@ class FusionTestRunner:
                 )
         return replace(action, value=resolver.resolve(action.value)), None
 
+    def _add_verifier_tokens(self, step: StepResult) -> None:
+        usage = getattr(self.verifier, "last_token_usage", None)
+        if isinstance(usage, TokenUsage):
+            step.token_usage += usage
+            step.verifier_tokens += usage
+
     def _write_step_log(
         self, goal: str, screen_text: str, action: str, outcome: str
     ) -> None:

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest/core/tokens.py

@@ -25,6 +25,12 @@ class TokenUsage:
             output_tokens=self.output_tokens + other.output_tokens,
         )
 
+    def __sub__(self, other: TokenUsage) -> TokenUsage:
+        return TokenUsage(
+            input_tokens=self.input_tokens - other.input_tokens,
+            output_tokens=self.output_tokens - other.output_tokens,
+        )
+
     def __iadd__(self, other: TokenUsage) -> TokenUsage:
         self.input_tokens += other.input_tokens
         self.output_tokens += other.output_tokens

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest/desktop/playwright_adapter.py

@@ -31,6 +31,14 @@ _ARIA_INLINE_RE = re.compile(r'^(\w+)\s*(?:\[[^\]]*\]\s*)*:\s*(.*\S)\s*$')
 _ARIA_BARE_RE = re.compile(r'^(\w+)\s*(?:\[[^\]]*\]\s*)*:?\s*$')
 
 
+# Upper bound for a model-requested "wait" action.
+_MAX_WAIT_SECONDS = 10.0
+
+
+class AdapterAborted(RuntimeError):
+    """The adapter was aborted (e.g. by the per-goal watchdog) mid-call."""
+
+
 class PlaywrightAdapter:
     """
     Adapter that drives a browser or Electron app with Playwright
@@ -68,6 +76,9 @@ class PlaywrightAdapter:
         self._page = None
         self._parser = ScreenParser()
         self._loop: asyncio.AbstractEventLoop | None = None
+        # abort() support: the adapter call in progress, and a close it started.
+        self._in_flight: asyncio.Task | None = None
+        self._closing: asyncio.Task | None = None
 
     # ── Lifecycle ──────────────────────────────────────────────────────────────
 
@@ -82,10 +93,42 @@ class PlaywrightAdapter:
 
     def stop(self) -> None:
         loop = self._get_loop()
+        # Let a close started by abort() finish before closing the loop.
+        if self._closing is not None and not self._closing.done():
+            loop.run_until_complete(self._closing)
+        self._closing = None
         loop.run_until_complete(self.async_stop())
         loop.close()
         self._loop = None
 
+    def abort(self) -> None:
+        """Interrupt the adapter from another thread (e.g. a watchdog) so the
+        call in flight fails right away: cancel it and close the browser
+        (Playwright calls like wait_for_selector only give up when their
+        browser goes away). stop() then finishes the close."""
+        loop = self._loop
+        if loop is None or not loop.is_running():
+            return
+
+        def _interrupt() -> None:
+            # Only our own call — Playwright's internal tasks must keep running.
+            if self._in_flight is not None:
+                self._in_flight.cancel()
+            self._closing = loop.create_task(self.async_stop())
+
+        loop.call_soon_threadsafe(_interrupt)
+
+    def _run(self, coro):
+        """Run an adapter coroutine on the adapter's loop, as a task abort() can cancel."""
+        loop = self._get_loop()
+        self._in_flight = loop.create_task(coro)
+        try:
+            return loop.run_until_complete(self._in_flight)
+        except asyncio.CancelledError as exc:
+            raise AdapterAborted("Playwright call aborted") from exc
+        finally:
+            self._in_flight = None
+
     async def async_start(self, url: str | None = None) -> None:
         try:
             from playwright.async_api import async_playwright
@@ -132,17 +175,23 @@ class PlaywrightAdapter:
         await self._page.wait_for_load_state("domcontentloaded")
 
     async def async_stop(self) -> None:
-        if self._browser:
-            await self._browser.close()
-        if self._playwright:
-            await self._playwright.stop()
+        """Close the browser and Playwright. Safe to call more than once."""
+        browser, self._browser = self._browser, None
+        playwright, self._playwright = self._playwright, None
+        for closer in (browser.close if browser else None, playwright.stop if playwright else None):
+            if closer is None:
+                continue
+            try:
+                await closer()
+            except Exception as exc:  # noqa: BLE001 — already closed / crashed
+                logger.debug("Ignoring error while stopping Playwright: %s", exc)
         logger.info("PlaywrightAdapter stopped")
 
     # ── Screen reading ─────────────────────────────────────────────────────────
 
     def site_links(self, limit: int = 10) -> list[str]:
         """Same-origin page URLs linked from the header/navigation (fallback: any link)."""
-        return self._get_loop().run_until_complete(self._async_site_links(limit))
+        return self._run(self._async_site_links(limit))
 
     async def _async_site_links(self, limit: int) -> list[str]:
         from urllib.parse import urldefrag, urljoin, urlsplit  # noqa: PLC0415
@@ -171,7 +220,7 @@ class PlaywrightAdapter:
 
     def navigate(self, url: str) -> None:
         """Navigate to a URL and wait for the page to load."""
-        self._get_loop().run_until_complete(self._async_navigate(url))
+        self._run(self._async_navigate(url))
 
     async def _async_navigate(self, url: str) -> None:
         # If the current page is closed/crashed, open a fresh one in the same context
@@ -195,7 +244,7 @@ class PlaywrightAdapter:
             pass
 
     def get_screen(self) -> tuple[list[UIElement], str]:
-        return self._get_loop().run_until_complete(self.async_get_screen())
+        return self._run(self.async_get_screen())
 
     async def async_get_screen(self) -> tuple[list[UIElement], str]:
         """Extract accessibility tree from the current page and parse it.
@@ -355,7 +404,7 @@ class PlaywrightAdapter:
         return nodes
 
     def get_screenshot(self) -> bytes:
-        return self._get_loop().run_until_complete(self.async_get_screenshot())
+        return self._run(self.async_get_screenshot())
 
     async def async_get_screenshot(self) -> bytes:
         return await self._page.screenshot(type="png")
@@ -363,9 +412,7 @@ class PlaywrightAdapter:
     # ── Action execution ───────────────────────────────────────────────────────
 
     def execute(self, action: Action, elements: list[UIElement]) -> bool:
-        return self._get_loop().run_until_complete(
-            self.async_execute(action, elements)
-        )
+        return self._run(self.async_execute(action, elements))
 
     async def async_execute(self, action: Action, elements: list[UIElement]) -> bool:
         """
@@ -384,8 +431,11 @@ class PlaywrightAdapter:
             elif action.action_type == "back":
                 await self._page.go_back()
             elif action.action_type == "wait":
-                secs = float(action.value) if action.value else 1.5
-                await asyncio.sleep(secs)
+                try:
+                    secs = float(action.value) if action.value else 1.5
+                except ValueError:
+                    secs = 1.5
+                await asyncio.sleep(min(max(secs, 0.0), _MAX_WAIT_SECONDS))
             elif action.action_type == "select":
                 await self._select(action, elements)
 

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest/discovery/goal_generator.py

@@ -147,6 +147,7 @@ _REQUEST = LLMRequest(
     empty="{}",
     # 20 goals of JSON plus the model's hidden reasoning/thinking.
     max_output_tokens=8192,
+    thinking=True,   # open-ended generation benefits from thinking
 )
 
 

{fusiontest-0.2.2 → fusiontest-0.2.4}/fusiontest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.2
+Version: 0.2.4
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.2 → fusiontest-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fusiontest"
-version = "0.2.2"
+version = "0.2.4"
 description = "AI-powered UI testing for mobile and desktop — by FusionLeap.io"
 readme = "README.md"
 requires-python = ">=3.10"