fusiontest 0.2.0__tar.gz → 0.2.2__tar.gz

{fusiontest-0.2.0 → fusiontest-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.0
+Version: 0.2.2
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.0 → fusiontest-0.2.2}/fusiontest/core/action_model.py

@@ -73,6 +73,7 @@ def _error_status_code(err: Exception) -> int | None:
 _BACKEND_SPECIFIC_STATUS_CODES = {401, 403, 404}
 _BACKEND_SPECIFIC_TOKENS = (
     "model_not_found", "does not exist", "do not have access",
+    "this model only supports", "is not supported for this model",   # wrong kind of model
     "invalid api key", "invalid_api_key", "is not set",
     "api key not valid", "api_key_invalid",   # Google returns these as HTTP 400
 )
@@ -185,52 +186,6 @@ class Action:
 
 ActionBackend = Literal["mpnet", "gpt4o", "claude", "gemini", "ollama", "groq"]
 
-# ── Gemini model auto-detection ───────────────────────────────────────────────
-
-_gemini_model_cache: str | None = None
-_GEMINI_PREFERRED = ["flash", "pro"]        # tier preference, newest date wins
-
-
-def _resolve_gemini_model(client) -> str:
-    """Return the best available Gemini model for this API key (cached).
-
-    Preference: newest flash > older flash > newest pro > older pro.
-    Sort is DESCENDING by name so '2.5' beats '2.0' beats '1.5'.
-
-    Accepts a ``google.genai.Client`` instance from the new unified SDK
-    (``pip install google-genai``). The deprecated ``google-generativeai``
-    package is no longer supported.
-    """
-    global _gemini_model_cache  # noqa: PLW0603
-    if _gemini_model_cache is not None:
-        return _gemini_model_cache
-
-    try:
-        all_models: list[str] = []
-        for m in client.models.list():
-            name = (getattr(m, "name", "") or "").replace("models/", "")
-            if not name or "gemini" not in name.lower():
-                continue
-            if not any(t in name.lower() for t in ("flash", "pro")):
-                continue
-            all_models.append(name)
-
-        # Newest flash first, then newest pro — reverse=True so '2.5' > '2.0' > '1.5'
-        flash = sorted([m for m in all_models if "flash" in m.lower()], reverse=True)
-        pro   = sorted([m for m in all_models if "flash" not in m.lower()], reverse=True)
-        ordered = flash + pro
-
-        if ordered:
-            _gemini_model_cache = ordered[0]
-            logger.info("Auto-selected Gemini model: %s", _gemini_model_cache)
-            return _gemini_model_cache
-    except Exception as exc:
-        logger.warning("Could not list Gemini models: %s — falling back to gemini-2.5-flash", exc)
-
-    _gemini_model_cache = "gemini-2.5-flash"
-    return _gemini_model_cache
-
-
 # ── Fallback-chain construction ───────────────────────────────────────────────
 
 # Model "" means "resolve the default for this backend at call time"
@@ -243,6 +198,11 @@ _DEFAULT_CHAIN_MODEL = ""
 GROQ_DEFAULT_MODEL = "openai/gpt-oss-120b"
 GROQ_FAST_MODEL = "openai/gpt-oss-20b"   # separate rate-limit bucket
 
+# Pinned Gemini model; GEMINI_MODEL overrides it. Not auto-selected from the
+# model list — names like *-tts, *-image or Interactions-only models sort
+# above the general-purpose one and reject normal requests.
+GEMINI_DEFAULT_MODEL = "gemini-2.5-flash"
+
 
 # Output-token cap for LLM calls. Reasoning models (Groq gpt-oss, Gemini 2.5)
 # spend part of this on hidden reasoning before any answer, so a tight cap
@@ -264,7 +224,7 @@ def default_model_for_backend(backend: str) -> str:
     if backend == "claude":
         return "claude-sonnet-4-6"
     if backend == "gemini":
-        return _DEFAULT_CHAIN_MODEL  # resolved lazily
+        return os.getenv("GEMINI_MODEL", GEMINI_DEFAULT_MODEL)
     if backend == "ollama":
         return os.getenv("OLLAMA_MODEL", "qwen2.5:7b")
     if backend == "groq":
@@ -334,7 +294,7 @@ def _build_default_fallback_chain(primary: str) -> list[tuple[str, str]]:
     seen_backends = {primary}
 
     if "gemini" not in seen_backends and os.getenv("GEMINI_API_KEY"):
-        chain.append(("gemini", _DEFAULT_CHAIN_MODEL))
+        chain.append(("gemini", default_model_for_backend("gemini")))
         seen_backends.add("gemini")
 
     if "groq" not in seen_backends and os.getenv("GROQ_API_KEY"):
@@ -360,6 +320,7 @@ class LLMRequest:
     reminder: str = ""          # appended to the prompt for backends without a JSON mode
     json_object: bool = False   # OpenAI-compatible response_format={"type": "json_object"}
     empty: str = ""             # returned when the model produces no text
+    max_output_tokens: int = LLM_MAX_OUTPUT_TOKENS  # includes hidden reasoning/thinking
 
 
 class LLMCaller:
@@ -381,6 +342,8 @@ class LLMCaller:
         self.temperature = temperature
         self.max_retries_per_backend = max_retries_per_backend
         self.backoff_seconds = backoff_seconds
+        # "backend:model" of the entry that answered the last call.
+        self.last_backend: str = ""
         # Per-(backend, model) client cache. Lazily populated.
         self._clients: dict[tuple[str, str], tuple[Any, str]] = {}
 
@@ -416,11 +379,8 @@ class LLMCaller:
     def _get_client(self, backend: str, model: str) -> tuple[Any, str]:
         """
         Lazily create (and cache) the client for a given backend+model.
-        Returns (client, resolved_model). The resolved_model may differ from
-        the requested `model` when Gemini auto-resolution is used — we cache
-        (client, resolved_model) together so subsequent calls keep using the
-        resolved name instead of the empty sentinel that `default_model_for_backend`
-        returns for Gemini.
+        Returns (client, resolved_model); an empty `model` means the backend's
+        default (see default_model_for_backend).
         """
         cache_key = (backend, model)
         if cache_key in self._clients:
@@ -493,12 +453,6 @@ class LLMCaller:
                     "export GEMINI_API_KEY=AIza..."
                 )
             client = genai.Client(api_key=api_key)
-            if not resolved_model:
-                resolved_model = _resolve_gemini_model(client)
-                logger.info(
-                    "Gemini client created; resolved model=%r (cache_key=%r)",
-                    resolved_model, cache_key,
-                )
 
         else:
             raise RuntimeError(f"Unknown backend: {backend}")
@@ -542,7 +496,9 @@ class LLMCaller:
 
             for attempt in range(self.max_retries_per_backend + 1):
                 try:
-                    return self._call_llm_single(prompt, backend, model, request)
+                    answer = self._call_llm_single(prompt, backend, model, request)
+                    self.last_backend = f"{backend}:{model or default_model_for_backend(backend)}"
+                    return answer
                 except Exception as err:  # noqa: BLE001
                     last_error = err
                     transient = _is_transient_error(err)
@@ -613,30 +569,6 @@ class LLMCaller:
         """
         client, resolved_model = self._get_client(backend, model)
 
-        # Defensive guard: the Gemini SDK raises a cryptic
-        # ``ValueError: model is required.`` when ``model=""`` reaches
-        # ``generate_content``. This has happened in prod when a cache path
-        # or future refactor lets the sentinel `_DEFAULT_CHAIN_MODEL` leak
-        # through. Force a re-resolve and raise a loud, actionable error if
-        # that still fails — it's better to fail the attempt with context
-        # than to hand an empty string to the SDK.
-        if backend == "gemini" and not resolved_model:
-            logger.error(
-                "Gemini resolved_model is empty after _get_client(backend=%r, model=%r). "
-                "Attempting emergency re-resolve.",
-                backend, model,
-            )
-            resolved_model = _resolve_gemini_model(client)
-            if not resolved_model:
-                raise RuntimeError(
-                    "Could not resolve a Gemini model name. "
-                    "This should never happen — _resolve_gemini_model falls back to "
-                    "'gemini-2.5-flash' on listing failures. Check for a corrupted "
-                    "_gemini_model_cache."
-                )
-            # Self-heal the cache so the next call doesn't hit this path.
-            self._clients[(backend, model)] = (client, resolved_model)
-
         if backend in ("gpt4o", "ollama", "groq"):
             resp = client.chat.completions.create(
                 model=resolved_model,
@@ -645,7 +577,7 @@ class LLMCaller:
                     {"role": "user", "content": prompt},
                 ],
                 temperature=self.temperature,
-                max_tokens=LLM_MAX_OUTPUT_TOKENS,
+                max_tokens=request.max_output_tokens,
                 **({"response_format": {"type": "json_object"}} if request.json_object else {}),
                 **openai_compat_extra_args(resolved_model),
             )
@@ -658,7 +590,7 @@ class LLMCaller:
         if backend == "claude":
             resp = client.messages.create(
                 model=resolved_model,
-                max_tokens=LLM_MAX_OUTPUT_TOKENS,
+                max_tokens=request.max_output_tokens,
                 system=request.system,
                 messages=[{"role": "user", "content": prompt + request.reminder}],
             )
@@ -676,7 +608,7 @@ class LLMCaller:
                 config=genai_types.GenerateContentConfig(
                     system_instruction=request.system,
                     temperature=self.temperature,
-                    max_output_tokens=LLM_MAX_OUTPUT_TOKENS,
+                    max_output_tokens=request.max_output_tokens,
                 ),
             )
             meta = getattr(resp, "usage_metadata", None)

{fusiontest-0.2.0 → fusiontest-0.2.2}/fusiontest/desktop/playwright_adapter.py

@@ -140,6 +140,31 @@ class PlaywrightAdapter:
 
     # ── Screen reading ─────────────────────────────────────────────────────────
 
+    def site_links(self, limit: int = 10) -> list[str]:
+        """Same-origin page URLs linked from the header/navigation (fallback: any link)."""
+        return self._get_loop().run_until_complete(self._async_site_links(limit))
+
+    async def _async_site_links(self, limit: int) -> list[str]:
+        from urllib.parse import urldefrag, urljoin, urlsplit  # noqa: PLC0415
+
+        hrefs: list[str] = await self._page.locator("header a[href], nav a[href]").evaluate_all(
+            "els => els.map(e => e.getAttribute('href'))"
+        )
+        if not hrefs:
+            hrefs = await self._page.locator("a[href]").evaluate_all(
+                "els => els.map(e => e.getAttribute('href'))"
+            )
+        base = self._page.url
+        origin = urlsplit(base).netloc
+        links: list[str] = []
+        for href in hrefs:
+            if not href or href.startswith(("mailto:", "tel:", "javascript:")):
+                continue
+            url = urldefrag(urljoin(base, href)).url.rstrip("/")
+            if urlsplit(url).netloc == origin and url != base.rstrip("/") and url not in links:
+                links.append(url)
+        return links[:limit]
+
     def current_url(self) -> str:
         """URL of the current page ("" before start). Used for secret site binding."""
         return self._page.url if self._page else ""

fusiontest-0.2.2/fusiontest/discovery/goal_generator.py

@@ -0,0 +1,287 @@
+"""
+fusiontest/discovery/goal_generator.py
+
+Autonomous goal discovery: propose test goals for a site from what it
+actually contains.
+
+1. ``snapshot_site`` opens the site in a headless browser and captures the
+   start page plus up to a few pages linked from its navigation, using the
+   same screen representation the runner uses (headings, text, form fields,
+   buttons).
+2. ``GoalGenerator.generate`` asks an LLM (through the shared fallback chain)
+   for goals, each quoting *evidence* from the captured pages.
+3. Goals whose evidence isn't on those pages are dropped — the model may not
+   invent features (login, search, consent checkboxes…) the site doesn't have.
+
+There is no template fallback: if no LLM can answer, generation fails with the
+underlying error rather than presenting generic goals as "discovered".
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Literal
+from urllib.parse import urlsplit
+
+from fusiontest.core.action_model import (
+    LLMCaller,
+    LLMRequest,
+    LLMUnavailableError,
+    _is_ollama_reachable,
+)
+from fusiontest.core.replay import select_evidence
+
+logger = logging.getLogger(__name__)
+
+GoalCategory = Literal["critical_path", "happy_path", "edge_cases"]
+_CATEGORIES = ("critical_path", "happy_path", "edge_cases")
+
+
+class GoalGenerationError(RuntimeError):
+    """Goals couldn't be generated (site unreachable, no LLM, unusable answer)."""
+
+
+@dataclass
+class DiscoveredGoal:
+    id: str
+    name: str                  # short 5-8 word title for the YAML output
+    text: str
+    category: GoalCategory
+    confidence: float          # 0.0 – 1.0
+    selected: bool = True      # user can deselect before running
+    edited: bool = False       # has the user overridden the text?
+    evidence: str = ""         # quote from the site content the goal relies on
+
+
+@dataclass
+class GoalGeneratorConfig:
+    max_goals: int = 20
+    max_pages: int = 8               # start page + pages linked from navigation
+    max_chars_per_page: int = 4000
+    temperature: float = 0.3
+
+
+@dataclass
+class SiteSnapshot:
+    url: str
+    pages: list[tuple[str, str]] = field(default_factory=list)   # (url, screen text)
+
+    @property
+    def content(self) -> str:
+        return "\n\n".join(f"=== PAGE {urlsplit(u).path or '/'} ===\n{text}" for u, text in self.pages)
+
+
+# ── Site snapshot ─────────────────────────────────────────────────────────────
+
+def snapshot_site(url: str, config: GoalGeneratorConfig | None = None) -> SiteSnapshot:
+    """Capture the start page and pages linked from its navigation."""
+    from fusiontest.desktop.playwright_adapter import PlaywrightAdapter  # noqa: PLC0415
+
+    cfg = config or GoalGeneratorConfig()
+    snapshot = SiteSnapshot(url=url)
+    adapter = PlaywrightAdapter(headless=True)
+    try:
+        adapter.start(url)
+        snapshot.pages.append((url, _page_text(adapter, cfg)))
+        for link in adapter.site_links(limit=cfg.max_pages - 1):
+            try:
+                adapter.navigate(link)
+                snapshot.pages.append((link, _page_text(adapter, cfg)))
+            except Exception as exc:  # noqa: BLE001 — one bad page shouldn't sink discovery
+                logger.warning("Discovery: skipping %s (%s)", link, exc)
+    except Exception as exc:  # noqa: BLE001
+        raise GoalGenerationError(f"Could not open {url}: {exc}") from exc
+    finally:
+        try:
+            adapter.stop()
+        except Exception:  # noqa: BLE001
+            pass
+    logger.info("Discovery snapshot of %s: %d page(s), %d chars",
+                url, len(snapshot.pages), len(snapshot.content))
+    return snapshot
+
+
+def _page_text(adapter, cfg: GoalGeneratorConfig) -> str:
+    _, screen = adapter.get_screen()
+    return screen[: cfg.max_chars_per_page]
+
+
+# ── Prompt ────────────────────────────────────────────────────────────────────
+
+_SYSTEM_PROMPT = """\
+You are a senior QA engineer writing automated UI test goals for a specific website.
+You are given the content of the site's pages as an accessibility snapshot:
+interactive elements (links, buttons, fields with their labels) and page text.
+
+Rules:
+- Only write goals for features that appear in the provided content. Do NOT
+  assume features that aren't shown — no login, sign-up, search, checkout,
+  account, or consent-checkbox goals unless those controls appear in the content.
+- Each goal is one complete, self-contained action sentence a tester can follow,
+  e.g. "Navigate to the Pricing page and verify all three plan tiers are displayed".
+- Refer to things by their visible text (link, button, heading or field label).
+  No CSS selectors, IDs, or code.
+- Do not submit real forms, send messages, or make purchases: this may be a
+  live production site. For forms, verify the fields and client-side validation
+  (e.g. required-field errors) instead of submitting successfully.
+- Cover navigation, key content, forms and validation — as far as the site has them.
+- evidence: a short EXACT quote (3-80 characters) copied from the provided
+  content that the goal depends on — a heading, link, button or field label.
+
+Return ONLY a JSON object: {"goals": [ ... ]}. Each goal has exactly these keys:
+  name       (short 5-8 word title)
+  text       (the full goal sentence)
+  category   ("critical_path" | "happy_path" | "edge_cases")
+  confidence (number 0.0-1.0: how clearly the content supports this goal)
+  evidence   (exact quote from the content)
+"""
+
+_REQUEST = LLMRequest(
+    system=_SYSTEM_PROMPT,
+    reminder='\n\nIMPORTANT: respond with ONLY the JSON object {"goals": [...]}. No other text.',
+    json_object=True,
+    empty="{}",
+    # 20 goals of JSON plus the model's hidden reasoning/thinking.
+    max_output_tokens=8192,
+)
+
+
+def _build_user_message(snapshot: SiteSnapshot, max_goals: int) -> str:
+    return (
+        f"Site: {snapshot.url}\n\n"
+        f"SITE CONTENT ({len(snapshot.pages)} page(s)):\n{snapshot.content}\n\n"
+        f"Write up to {max_goals} test goals for this site, grounded in the content above, "
+        f"across critical_path (the site's most important flows), happy_path (navigation "
+        f"and features) and edge_cases (validation and error states)."
+    )
+
+
+# ── Parsing ───────────────────────────────────────────────────────────────────
+
+def _derive_name(text: str) -> str:
+    """Fallback: condense a goal sentence into a short 5-8 word title."""
+    cleaned = re.sub(
+        r"^(verify\s+(that\s+)?|navigate\s+to\s+(the\s+)?|check\s+(that\s+)?|"
+        r"attempt\s+to\s+|click\s+|submit\s+|open\s+(the\s+)?|ensure\s+(that\s+)?)",
+        "",
+        text,
+        flags=re.I,
+    ).strip()
+    words = cleaned.split()
+    name = " ".join(words[:7])
+    if len(words) > 7:
+        name += "…"
+    return name.capitalize()
+
+
+def _parse_goals(raw: str, max_goals: int, content: str) -> list[DiscoveredGoal]:
+    """Parse the LLM's JSON and keep only goals whose evidence is in the content."""
+    raw = re.sub(r"^```[a-z]*\n?|\n?```$", "", raw.strip(), flags=re.M)
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise GoalGenerationError(f"The model's answer wasn't valid JSON: {exc}") from exc
+    items = data if isinstance(data, list) else data.get("goals", [])
+
+    goals: list[DiscoveredGoal] = []
+    dropped: list[str] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        text = str(item.get("text", "")).strip()
+        if not text:
+            continue
+        evidence = select_evidence([str(item.get("evidence", ""))], content, limit=1)
+        if not evidence:
+            dropped.append(text)
+            continue
+        category = item.get("category", "happy_path")
+        if category not in _CATEGORIES:
+            category = "happy_path"
+        try:
+            confidence = max(0.0, min(1.0, float(item.get("confidence", 0.8))))
+        except (TypeError, ValueError):
+            confidence = 0.8
+        goals.append(DiscoveredGoal(
+            id=f"goal-{len(goals):03d}",
+            name=str(item.get("name", "")).strip() or _derive_name(text),
+            text=text,
+            category=category,  # type: ignore[arg-type]
+            confidence=confidence,
+            selected=confidence >= 0.75,
+            evidence=evidence[0],
+        ))
+        if len(goals) >= max_goals:
+            break
+
+    if dropped:
+        logger.info("Dropped %d goal(s) not grounded in the site content: %s",
+                    len(dropped), "; ".join(d[:60] for d in dropped))
+    return goals
+
+
+# ── Backend choice ────────────────────────────────────────────────────────────
+
+_BACKEND_KEYS = {"groq": "GROQ_API_KEY", "claude": "ANTHROPIC_API_KEY",
+                 "gemini": "GEMINI_API_KEY", "gpt4o": "OPENAI_API_KEY"}
+_BACKEND_ALIASES = {"openai": "gpt4o"}
+
+
+def _primary_backend(preferred: str | None) -> str:
+    """The user's backend if usable, else the first one with credentials."""
+    choice = (preferred or "").strip().lower()
+    choice = _BACKEND_ALIASES.get(choice, choice)
+    if choice in _BACKEND_KEYS and os.getenv(_BACKEND_KEYS[choice]):
+        return choice
+    if choice == "ollama" and _is_ollama_reachable():
+        return "ollama"
+    for backend, key in _BACKEND_KEYS.items():
+        if os.getenv(key):
+            return backend
+    if _is_ollama_reachable():
+        return "ollama"
+    raise GoalGenerationError(
+        "No LLM configured: set GROQ_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY or "
+        "OPENAI_API_KEY (or run Ollama)."
+    )
+
+
+# ── Public interface ──────────────────────────────────────────────────────────
+
+class GoalGenerator:
+    """Generates prioritised, site-specific test goals for a given URL."""
+
+    def __init__(self, config: GoalGeneratorConfig | None = None) -> None:
+        self.config = config or GoalGeneratorConfig()
+        # Set after generate(): "backend:model" that produced the goals.
+        self.last_backend_used: str = ""
+
+    def generate(
+        self,
+        url: str,
+        preferred_backend: str | None = None,
+        snapshot: SiteSnapshot | None = None,
+    ) -> list[DiscoveredGoal]:
+        """Goals grounded in the site's content. Raises GoalGenerationError."""
+        snapshot = snapshot or snapshot_site(url, self.config)
+        if not snapshot.content.strip():
+            raise GoalGenerationError(f"No content could be read from {url}")
+
+        caller = LLMCaller(_primary_backend(preferred_backend), temperature=self.config.temperature)
+        try:
+            raw, _ = caller._call_llm(_build_user_message(snapshot, self.config.max_goals), _REQUEST)
+        except LLMUnavailableError as exc:
+            raise GoalGenerationError(str(exc)) from exc
+        self.last_backend_used = caller.last_backend
+
+        goals = _parse_goals(raw, self.config.max_goals, snapshot.content)
+        if not goals:
+            raise GoalGenerationError(
+                "The model didn't propose any goals grounded in the site's content."
+            )
+        logger.info("Generated %d goal(s) for %s via %s", len(goals), url, self.last_backend_used)
+        return goals

{fusiontest-0.2.0 → fusiontest-0.2.2}/fusiontest.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusiontest
-Version: 0.2.0
+Version: 0.2.2
 Summary: AI-powered UI testing for mobile and desktop — by FusionLeap.io
 Author-email: FusionLeap <hello@fusionleap.io>
 License: MIT

{fusiontest-0.2.0 → fusiontest-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fusiontest"
-version = "0.2.0"
+version = "0.2.2"
 description = "AI-powered UI testing for mobile and desktop — by FusionLeap.io"
 readme = "README.md"
 requires-python = ">=3.10"