fusiontest 0.1.0__py3-none-any.whl

fusiontest/__init__.py

@@ -0,0 +1,2 @@
+"""FusionTest — AI-powered UI testing for mobile and desktop."""
+__version__ = "0.1.0"

fusiontest/cli.py

@@ -0,0 +1,477 @@
+"""
+fusiontest/cli.py
+
+The `fusiontest` CLI — run tests from the command line or CI.
+
+Commands:
+  fusiontest run    — run a test goal or goal file
+  fusiontest demo   — run a pre-built demo against a public site (no app needed)
+  fusiontest init   — scaffold a fusiontest.config.yaml in the current directory
+  fusiontest report — print the last run result
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import click
+import yaml
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="0.1.0", prog_name="FusionTest")
+def main():
+    """FusionTest — AI-powered UI testing for mobile and desktop.\n\nby FusionLeap.io"""
+    pass
+
+
+# ── fusiontest run ──────────────────────────────────────────────────────────────
+
+@main.command()
+@click.option("--goal", "-g", multiple=True, help="Natural language test goal (repeatable)")
+@click.option("--file", "-f", "goal_file", type=click.Path(exists=True), help="YAML file of goals")
+@click.option("--platform", "-p",
+              type=click.Choice(["android", "ios", "playwright", "electron"]),
+              default="playwright", show_default=True)
+@click.option("--app", type=click.Path(), help="Path to .apk / .ipa / Electron binary")
+@click.option("--url", help="URL to open (for playwright/web targets)")
+@click.option("--locale", default="en-US", show_default=True, help="Locale for the test run")
+@click.option("--backend", type=click.Choice(["gpt4o", "claude", "mpnet"]),
+              default="gpt4o", show_default=True, help="AI model backend")
+@click.option("--headless/--no-headless", default=False, show_default=True)
+@click.option("--output", "-o", type=click.Path(), help="Write JSON report to this path")
+@click.option("--config", "config_file", type=click.Path(exists=True),
+              default="fusiontest.config.yaml", help="Config file path")
+def run(goal, goal_file, platform, app, url, locale, backend, headless, output, config_file):
+    """Run a FusionTest test — provide goals via --goal or --file."""
+
+    from dotenv import load_dotenv
+    load_dotenv(override=True)
+
+    # Load config file if it exists
+    cfg = {}
+    if config_file and Path(config_file).exists():
+        with open(config_file) as f:
+            cfg = yaml.safe_load(f) or {}
+
+    # Resolve goals
+    goals = list(goal)
+    if goal_file:
+        with open(goal_file) as f:
+            data = yaml.safe_load(f)
+        goals = data.get("goals", [])
+        platform = data.get("platform", platform)
+        locale = data.get("locale", locale)
+        if not url and data.get("url"):
+            url = data["url"]
+
+    if not goals:
+        console.print("[red]Error:[/red] No goals provided. Use --goal or --file.")
+        sys.exit(1)
+
+    # Print header
+    console.print()
+    console.print(Panel.fit(
+        f"[bold]FusionTest[/bold] by FusionLeap.io\n"
+        f"Platform: [cyan]{platform}[/cyan]  ·  Backend: [cyan]{backend}[/cyan]  ·  Locale: [cyan]{locale}[/cyan]",
+        border_style="blue",
+    ))
+    console.print(f"\n[bold]Goals ({len(goals)}):[/bold]")
+    for i, g in enumerate(goals, 1):
+        console.print(f"  {i}. {g}")
+    console.print()
+
+    # Build adapter
+    adapter = _build_adapter(platform, app, url, headless)
+
+    # Build runner
+    from fusiontest.core.runner import FusionTestRunner, RunnerConfig
+    from fusiontest.guardrails.engine import GuardrailsConfig
+
+    runner_cfg = RunnerConfig(
+        backend=backend,
+        guardrails=GuardrailsConfig(
+            loop_window=cfg.get("guardrails", {}).get("loop_detection_window", 5),
+            max_invalid_retries=cfg.get("guardrails", {}).get("max_retries", 3),
+        ),
+        screenshot_on_failure=True,
+    )
+    runner = FusionTestRunner(adapter, runner_cfg)
+
+    # Run with live progress
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console,
+    ) as progress:
+        progress.add_task("Running tests...", total=None)
+        time.time()
+        try:
+            result = runner.run(goals=goals, platform=platform, locale=locale)
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Interrupted by user[/yellow]")
+            sys.exit(1)
+        finally:
+            progress.stop()
+
+    # Print results table
+    _print_results(result)
+
+    # Save JSON report
+    if output:
+        _save_report(result, output)
+        console.print(f"\n[dim]Report saved to: {output}[/dim]")
+
+    sys.exit(0 if result.success else 1)
+
+
+# ── fusiontest demo ─────────────────────────────────────────────────────────────
+
+# Maps demo target names → (goal file path relative to package root, description)
+_DEMOS: dict[str, tuple[str, str]] = {
+    "dashboard":   ("tests/goals/demos/fusiontest_dashboard.yaml",
+                    "FusionTest dashboard self-test (requires: fusiontest-api + npm run dev)"),
+    "todomvc":     ("tests/goals/demos/todomvc.yaml",
+                    "TodoMVC — full CRUD flow on the React TodoMVC demo app"),
+    "wikipedia":   ("tests/goals/demos/wikipedia.yaml",
+                    "Wikipedia — search, navigate, and read an article"),
+    "hackernews":  ("tests/goals/demos/hacker_news.yaml",
+                    "Hacker News — front page browsing and comment navigation"),
+    "github":      ("tests/goals/demos/github.yaml",
+                    "GitHub — browse the fusion-test public repo (no login)"),
+    "httpbin":     ("tests/goals/demos/httpbin.yaml",
+                    "httpbin.org — HTTP form submission and response inspection"),
+}
+
+
+@main.command()
+@click.argument("target", required=False)
+@click.option("--backend", type=click.Choice(["gpt4o", "claude", "mpnet"]),
+              default="gpt4o", show_default=True)
+@click.option("--headless/--no-headless", default=False, show_default=True)
+@click.option("--output", "-o", type=click.Path(), help="Write JSON report to this path")
+@click.option("--list", "list_demos", is_flag=True, help="List all available demo targets")
+def demo(target, backend, headless, output, list_demos):
+    """Run a pre-built demo against a public site — no app setup required.
+
+    \b
+    Available targets:
+      dashboard   FusionTest dashboard self-test
+      todomvc     TodoMVC React CRUD flow
+      wikipedia   Wikipedia search and navigation
+      hackernews  Hacker News browsing
+      github      GitHub public repo browsing
+      httpbin     httpbin.org form submission
+
+    \b
+    Examples:
+      fusiontest demo todomvc
+      fusiontest demo wikipedia --backend claude
+      fusiontest demo --list
+    """
+    from dotenv import load_dotenv
+    load_dotenv()
+
+    if list_demos or not target:
+        console.print()
+        console.print(Panel.fit(
+            "[bold]FusionTest Demo Targets[/bold]\n"
+            "Run any of these with: [cyan]fusiontest demo <target>[/cyan]",
+            border_style="blue",
+        ))
+        console.print()
+        table = Table(show_header=True, header_style="bold", box=None, pad_edge=False)
+        table.add_column("Target", style="cyan", width=14)
+        table.add_column("Description")
+        for name, (_, desc) in _DEMOS.items():
+            table.add_row(name, desc)
+        console.print(table)
+        console.print()
+        if not target:
+            sys.exit(0)
+
+    target = target.lower().replace("-", "").replace("_", "")
+    # Allow partial matches (e.g. "todo" → "todomvc", "hack" → "hackernews")
+    matched = [k for k in _DEMOS if k.startswith(target)]
+    if not matched:
+        console.print(f"[red]Unknown demo target:[/red] {target}")
+        console.print("Run [cyan]fusiontest demo --list[/cyan] to see all available targets.")
+        sys.exit(1)
+    if len(matched) > 1:
+        console.print(f"[yellow]Ambiguous target '{target}':[/yellow] matches {matched}")
+        sys.exit(1)
+
+    demo_key = matched[0]
+    goal_rel_path, description = _DEMOS[demo_key]
+
+    # Resolve path: first relative to repo root, then as an installed package resource
+    repo_root = Path(__file__).parent.parent
+    goal_path = repo_root / goal_rel_path
+    if not goal_path.exists():
+        console.print(f"[red]Demo goal file not found:[/red] {goal_path}")
+        sys.exit(1)
+
+    with open(goal_path) as f:
+        data = yaml.safe_load(f)
+
+    goals    = data.get("goals", [])
+    platform = data.get("platform", "playwright")
+    locale   = data.get("locale", "en-US")
+    url      = data.get("url")
+    name     = data.get("name", description)
+
+    # Dashboard self-test: warn if services aren't running
+    if demo_key == "dashboard":
+        console.print()
+        console.print(Panel(
+            "[bold yellow]Dashboard self-test requires two services to be running:[/bold yellow]\n\n"
+            "  Terminal 1:  [cyan]fusiontest-api[/cyan]\n"
+            "  Terminal 2:  [cyan]cd dashboard && npm run dev[/cyan]\n\n"
+            "Then press Enter to continue, or Ctrl-C to cancel.",
+            border_style="yellow",
+        ))
+        try:
+            input()
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Cancelled.[/yellow]")
+            sys.exit(0)
+
+    console.print()
+    console.print(Panel.fit(
+        f"[bold]FusionTest Demo[/bold] — {name}\n"
+        f"Platform: [cyan]{platform}[/cyan]  ·  Backend: [cyan]{backend}[/cyan]  ·  "
+        f"URL: [cyan]{url or '(none)'}[/cyan]",
+        border_style="blue",
+    ))
+    console.print(f"\n[bold]Goals ({len(goals)}):[/bold]")
+    for i, g in enumerate(goals, 1):
+        console.print(f"  {i}. {g}")
+    console.print()
+
+    adapter = _build_adapter(platform, None, url, headless)
+
+    from fusiontest.core.runner import FusionTestRunner, RunnerConfig
+    runner = FusionTestRunner(adapter, RunnerConfig(backend=backend))
+
+    with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"),
+                  console=console) as progress:
+        progress.add_task(f"Running {demo_key} demo...", total=None)
+        try:
+            result = runner.run(goals=goals, name=name, platform=platform, locale=locale)
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Interrupted.[/yellow]")
+            sys.exit(1)
+        finally:
+            progress.stop()
+
+    _print_results(result)
+
+    if output:
+        _save_report(result, output)
+        console.print(f"\n[dim]Report saved to: {output}[/dim]")
+
+    if result.success:
+        console.print(
+            f"\n[green]✓ Demo passed![/green] FusionTest successfully navigated "
+            f"[bold]{demo_key}[/bold] using the [bold]{backend}[/bold] backend.\n"
+            f"Run [cyan]fusiontest demo --list[/cyan] to try another target."
+        )
+    else:
+        console.print(
+            "\n[red]✗ Demo had failures.[/red] Check the output above for details.\n"
+            "Tip: re-run with [cyan]--no-headless[/cyan] to watch the browser."
+        )
+
+    sys.exit(0 if result.success else 1)
+
+
+# ── fusiontest init ─────────────────────────────────────────────────────────────
+
+@main.command()
+def init():
+    """Scaffold a fusiontest.config.yaml in the current directory."""
+    config_path = Path("fusiontest.config.yaml")
+    if config_path.exists():
+        if not click.confirm(f"{config_path} already exists. Overwrite?"):
+            return
+
+    example_goals_path = Path("tests/goals/smoke.yaml")
+    example_goals_path.parent.mkdir(parents=True, exist_ok=True)
+
+    config = {
+        "model": {
+            "backbone": "gpt4o",
+            "max_steps_per_goal": 25,
+        },
+        "guardrails": {
+            "loop_detection": True,
+            "loop_detection_window": 5,
+            "max_retries": 3,
+            "backtrack_on_invalid": True,
+        },
+        "devices": {
+            "android": {"farm": "local"},
+            "desktop": {
+                "vision_fallback": True,
+                "playwright_for_electron": True,
+            },
+        },
+        "reporting": {
+            "save_recordings": True,
+        },
+    }
+
+    with open(config_path, "w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+
+    smoke_goals = {
+        "name": "Smoke test",
+        "platform": "playwright",
+        "locale": "en-US",
+        "url": "https://your-app-url.com",
+        "goals": [
+            "Verify the page loads and the main navigation is visible",
+            "Click the login button and verify the login form appears",
+        ],
+    }
+    with open(example_goals_path, "w") as f:
+        yaml.dump(smoke_goals, f, default_flow_style=False, sort_keys=False)
+
+    console.print(f"[green]✓[/green] Created {config_path}")
+    console.print(f"[green]✓[/green] Created {example_goals_path}")
+    console.print("\nNext steps:")
+    console.print("  1. Edit fusiontest.config.yaml to set your model backend")
+    console.print("  2. Set OPENAI_API_KEY or ANTHROPIC_API_KEY in .env")
+    console.print("  3. Run: [bold]fusiontest run --file tests/goals/smoke.yaml[/bold]")
+
+
+# ── fusiontest report ───────────────────────────────────────────────────────────
+
+@main.command()
+@click.argument("report_file", type=click.Path(exists=True))
+def report(report_file):
+    """Pretty-print a JSON report from a previous run."""
+    with open(report_file) as f:
+        data = json.load(f)
+    _print_report_data(data)
+
+
+# ── Helpers ────────────────────────────────────────────────────────────────────
+
+def _build_adapter(platform: str, app: str | None, url: str | None, headless: bool):
+    if platform == "playwright":
+        from fusiontest.desktop.playwright_adapter import PlaywrightAdapter
+        adapter = PlaywrightAdapter(browser_type="chromium", headless=headless)
+        adapter.start(url=url)
+        return adapter
+
+    elif platform == "electron":
+        if not app:
+            console.print("[red]--app is required for electron platform[/red]")
+            sys.exit(1)
+        from fusiontest.desktop.playwright_adapter import PlaywrightAdapter
+        adapter = PlaywrightAdapter(electron_app_path=app, headless=headless)
+        adapter.start()
+        return adapter
+
+    elif platform == "android":
+        from fusiontest.mobile.android_adapter import AndroidAdapter
+        caps = {}
+        if app:
+            caps["app"] = app
+        adapter = AndroidAdapter(desired_caps=caps)
+        adapter.start()
+        return adapter
+
+    elif platform == "ios":
+        from fusiontest.mobile.ios_adapter import IOSAdapter
+        caps = {}
+        if app:
+            caps["app"] = app
+        adapter = IOSAdapter(desired_caps=caps)
+        adapter.start()
+        return adapter
+
+    else:
+        console.print(f"[red]Unknown platform: {platform}[/red]")
+        sys.exit(1)
+
+
+def _print_results(result) -> None:
+    status = "[green]PASS ✓[/green]" if result.success else "[red]FAIL ✗[/red]"
+    console.print(f"\n{'─'*50}")
+    console.print(f"Result: {status}  Stability: [bold]{result.stability_score:.0%}[/bold]  "
+                  f"Duration: {result.total_duration_seconds:.1f}s")
+    console.print()
+
+    table = Table(show_header=True, header_style="bold", box=None, pad_edge=False)
+    table.add_column("#", style="dim", width=3)
+    table.add_column("Goal", min_width=30)
+    table.add_column("Status", width=8)
+    table.add_column("Steps", width=6)
+    table.add_column("Time", width=7)
+
+    for i, step in enumerate(result.steps, 1):
+        status_cell = "[green]PASS[/green]" if step.success else "[red]FAIL[/red]"
+        error_note = f"\n  [dim red]{step.error}[/dim red]" if step.error else ""
+        table.add_row(
+            str(i),
+            step.goal + error_note,
+            status_cell,
+            str(step.steps_taken),
+            f"{step.duration_seconds:.1f}s",
+        )
+
+    console.print(table)
+
+
+def _save_report(result, path: str) -> None:
+    data = {
+        "name": result.name,
+        "platform": result.platform,
+        "locale": result.locale,
+        "success": result.success,
+        "stability_score": result.stability_score,
+        "total_duration_seconds": result.total_duration_seconds,
+        "steps": [
+            {
+                "goal": s.goal,
+                "success": s.success,
+                "steps_taken": s.steps_taken,
+                "actions": s.actions,
+                "error": s.error,
+                "duration_seconds": s.duration_seconds,
+            }
+            for s in result.steps
+        ],
+    }
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)
+
+
+def _print_report_data(data: dict) -> None:
+    console.print(Panel.fit(
+        f"[bold]{data['name']}[/bold]\n"
+        f"Platform: {data['platform']}  ·  Locale: {data['locale']}  ·  "
+        f"Stability: {data['stability_score']:.0%}",
+        border_style="blue" if data["success"] else "red",
+    ))
+    for _i, step in enumerate(data.get("steps", []), 1):
+        icon = "✓" if step["success"] else "✗"
+        color = "green" if step["success"] else "red"
+        console.print(f"  [{color}]{icon}[/{color}] {step['goal']}  [{step['steps_taken']} steps, {step['duration_seconds']:.1f}s]")
+        if step.get("error"):
+            console.print(f"     [dim red]{step['error']}[/dim red]")
+
+
+if __name__ == "__main__":
+    main()