screenforge 0.4.0__py3-none-any.whl

cli/modes/action.py

@@ -0,0 +1,262 @@
+"""Single-action execution mode."""
+
+import json
+import os
+import sys
+
+import cli.shared as _shared
+from cli.reporter import (
+    _apply_resume_summary,
+    _build_action_summary,
+    _build_inline_action_data,
+    _build_reporter,
+    _emit_run_started,
+)
+from cli.shared import (
+    _capture_ui_state,
+    _connect_adapter,
+    _ensure_executor_runtime,
+    _ensure_history_manager,
+    _ensure_ui_compressors,
+    _SharedAdapterManager,
+    current_url,
+    get_initial_header,
+    log,
+    save_to_disk,
+)
+from common.failure_diagnosis import diagnose
+from common.runtime_modes import MODE_RUN
+
+
+def build_failure_payload(
+    *,
+    action_name: str,
+    platform: str,
+    assertion_failed: bool,
+    error_code: str,
+    locator_value: str,
+    ui_tree: dict,
+    current_url: str,
+) -> dict:
+    """Assemble the --action --json failure payload.
+
+    A failed assertion is a verdict: keep the original bare shape (no diagnosis,
+    no page, no retry bait). An engine_error is where the agent needs help, so
+    enrich it with the did-you-mean diagnosis, the current ui_tree, and the URL.
+
+    Note: ui_tree, element_count, and current_url are emitted ONLY for
+    engine_error — the assertion_failed path returns the bare verdict above.
+    """
+    payload = {
+        "ok": False,
+        "action": action_name,
+        "platform": platform,
+        "result": "assertion_failed" if assertion_failed else "engine_error",
+        "assertion_failed": assertion_failed,
+        "error": (
+            f"Assertion failed: {action_name}"
+            if assertion_failed
+            else f"Action failed: {action_name}"
+        ),
+    }
+    if assertion_failed:
+        return payload
+
+    diag = diagnose(
+        error_code=error_code or "E037",
+        locator_value=locator_value or "",
+        ui_elements=ui_tree.get("ui_elements", []) or [],
+    )
+    payload.update(diag.to_dict())
+    payload["ui_tree"] = ui_tree
+    payload["element_count"] = len(ui_tree.get("ui_elements", []) or [])
+    payload["current_url"] = current_url
+    return payload
+
+
+def run_action_default_mode(
+    args,
+    output_script_path: str,
+    resume_context: dict,
+    shared_adapter_manager: _SharedAdapterManager | None = None,
+) -> int:
+    adapter = None
+    owns_adapter = False
+    json_mode = args.json
+    reporter = _build_reporter(args, output_script_path, MODE_RUN, json_output=False)
+    exit_code = 1
+    final_status = "failed"
+    final_error = ""
+    steps_executed = 0
+    _emit_run_started(reporter, args, output_script_path, MODE_RUN)
+    _apply_resume_summary(reporter, resume_context)
+
+    try:
+        action_data = _build_inline_action_data(args)
+        action_summary = _build_action_summary(args, action_data, executed_steps=0)
+        reporter.update_control_summary(
+            control_kind="action",
+            control_label=action_summary["action_name"],
+            source_ref="inline://action",
+            action=action_summary["action"],
+            locator_type=action_summary["locator_type"],
+            locator_value=action_summary["locator_value"],
+            extra_value=action_summary["extra_value"],
+        )
+        reporter.emit_event(
+            "action_loaded",
+            action_name=action_summary["action_name"],
+            action=action_summary["action"],
+            locator_type=action_summary["locator_type"],
+            locator_value=action_summary["locator_value"],
+        )
+
+        try:
+            if shared_adapter_manager:
+                adapter = shared_adapter_manager.get_or_create(args.platform, args.env)
+            else:
+                adapter = _connect_adapter(args, reporter)
+                owns_adapter = True
+            device = adapter.driver
+        except Exception as e:
+            final_error = str(e)
+            reporter.emit_event("startup_failed", platform=args.platform, error=str(e))
+            log.error(f"❌ [Error] {args.platform} connection failed: {e}")
+            return 1
+
+        _ensure_history_manager()
+        _ensure_executor_runtime()
+
+        if os.path.exists(output_script_path):
+            with open(output_script_path, "r", encoding="utf-8") as f:
+                existing_lines = f.readlines()
+            history_manager = _shared.StepHistoryManager(initial_content=existing_lines)
+            log.info(f"📎 [System] Append mode: continuing on existing script ({output_script_path})")
+        else:
+            # Name the test after the action so the file is discoverable.
+            header = get_initial_header(label=action_data.get("name") or None)
+            history_manager = _shared.StepHistoryManager(initial_content=header)
+            save_to_disk(output_script_path, header)
+
+        if shared_adapter_manager:
+            # Reuse the session's shared executor so a `ref @N` action resolves
+            # against the cache a prior inspect_ui populated (same MCP session).
+            executor = shared_adapter_manager.get_executor(args.platform, args.env)
+        else:
+            executor = _shared.UIExecutor(device, platform=args.platform)
+
+        reporter.emit_event(
+            "step_started",
+            step=1,
+            source="action",
+            step_name=action_data["name"],
+        )
+        result = executor.execute_and_record(action_data)
+        if not result.get("success"):
+            assertion_failed = bool(result.get("assertion_failed"))
+            if assertion_failed:
+                final_error = f"Assertion failed: {action_data['name']}"
+            else:
+                final_error = f"Action failed: {action_data['name']}"
+            reporter.emit_event(
+                "action_executed",
+                step=1,
+                success=False,
+                action_description=action_data["name"],
+            )
+            log.error(f"❌ [Action] Failed: {action_data['name']}")
+            if json_mode:
+                # Distinguish a verification verdict (assertion_failed=true: the
+                # SUT did not meet the assertion — do NOT retry) from a real
+                # engine error. Only engine errors get the did-you-mean
+                # diagnosis + current ui_tree (the agent needs the page to
+                # recover); a verdict stays a bare result so we never bait a
+                # retry on a legitimately-failed assertion.
+                # Capture the URL at the moment of failure, BEFORE compress_web_dom
+                # traverses the DOM (a mid-redirect click could otherwise settle the
+                # navigation and report the post-redirect URL).
+                page_url = current_url(adapter, args.platform)
+                ui_tree = {}
+                if not assertion_failed:
+                    _ensure_ui_compressors()
+                    try:
+                        ui_json, _ = _capture_ui_state(args, adapter, reporter, 1)
+                        ui_tree = json.loads(ui_json)
+                    except Exception as e:
+                        # Degrade (connection lost / non-JSON) but don't hide it:
+                        # a silent empty tree yields empty candidates with no signal.
+                        log.warning(f"⚠️ [Action] Failed to capture post-failure UI state: {e}")
+                        ui_tree = {}
+                error_code = result.get("error_code", "") or ""
+                if not error_code and not assertion_failed:
+                    log.warning("⚠️ [Action] executor returned no error_code; defaulting to E037")
+                payload = build_failure_payload(
+                    action_name=action_data["name"],
+                    platform=args.platform,
+                    assertion_failed=assertion_failed,
+                    error_code=error_code,
+                    locator_value=getattr(args, "locator_value", "") or "",
+                    ui_tree=ui_tree,
+                    current_url=page_url,
+                )
+                json.dump(payload, sys.stdout, ensure_ascii=False)
+                sys.stdout.write("\n")
+                sys.stdout.flush()
+            return 1
+
+        history_manager.add_step(result["code_lines"], result["action_description"])
+        save_to_disk(output_script_path, history_manager.get_current_file_content())
+        reporter.emit_event(
+            "action_executed",
+            step=1,
+            success=True,
+            action_description=result["action_description"],
+        )
+        steps_executed = 1
+        reporter.update_summary(
+            action_summary=_build_action_summary(
+                args, action_data, executed_steps=steps_executed
+            )
+        )
+        reporter.update_control_summary(executed_steps=steps_executed)
+        final_status = "success"
+        exit_code = 0
+
+        if json_mode:
+            _ensure_ui_compressors()
+            ui_json, _ = _capture_ui_state(args, adapter, reporter, 1)
+            try:
+                ui_tree = json.loads(ui_json)
+            except (json.JSONDecodeError, TypeError):
+                ui_tree = {}
+            json.dump({
+                "ok": True,
+                "action": action_data["name"],
+                "platform": args.platform,
+                "ui_tree": ui_tree,
+                "element_count": len(ui_tree.get("ui_elements", [])),
+                "output_script": output_script_path,
+                "current_url": current_url(adapter, args.platform),
+            }, sys.stdout, ensure_ascii=False)
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+
+        return 0
+    except Exception as e:
+        final_error = str(e)
+        reporter.emit_event("action_run_failed", error=str(e))
+        log.error(f"❌ [Action] Failed: {e}")
+        return 1
+    finally:
+        reporter.finalize(
+            status=final_status,
+            exit_code=exit_code,
+            steps_executed=steps_executed,
+            last_error=final_error,
+        )
+        log.info(f"🏁 Done. Generated script saved to: {output_script_path}")
+        if owns_adapter and adapter:
+            try:
+                adapter.teardown()
+            except Exception as e:
+                log.warning(f"⚠️ [Warning] Cleanup failed: {e}")

cli/modes/default.py

@@ -0,0 +1,248 @@
+"""Default autonomous brain loop mode."""
+
+import cli.shared as _shared
+from cli.reporter import (
+    _apply_resume_summary,
+    _build_reporter,
+    _emit_run_started,
+)
+from cli.shared import (
+    _capture_ui_state,
+    _connect_adapter,
+    _ensure_executor_runtime,
+    _ensure_history_manager,
+    _ensure_runtime_classes,
+    get_initial_header,
+    log,
+    save_to_disk,
+)
+from common.runtime_modes import MODE_RUN
+
+
+def run_default_mode(
+    args,
+    output_script_path: str,
+    context_content: str,
+    resume_context: dict,
+) -> int:
+    adapter = None
+    reporter = _build_reporter(args, output_script_path, MODE_RUN)
+    exit_code = 1
+    final_status = "failed"
+    final_error = ""
+    steps_executed = 0
+    _emit_run_started(reporter, args, output_script_path, MODE_RUN)
+    _apply_resume_summary(reporter, resume_context)
+
+    try:
+        try:
+            adapter = _connect_adapter(args, reporter)
+            device = adapter.driver
+        except Exception as e:
+            final_error = str(e)
+            reporter.emit_event("startup_failed", platform=args.platform, error=str(e))
+            log.error(f"❌ [Error] {args.platform} connection failed: {e}")
+            return 1
+
+        _ensure_history_manager()
+        _ensure_executor_runtime()
+        header = get_initial_header(label=str(getattr(args, "goal", "")).strip() or None)
+        history_manager = _shared.StepHistoryManager(initial_content=header)
+        save_to_disk(output_script_path, header)
+
+        _ensure_runtime_classes()
+        brain = _shared.AutonomousBrain()
+        executor = _shared.UIExecutor(device, platform=args.platform)
+
+        step_count = 0
+        last_error = ""
+        consecutive_failures = 0
+        last_ui_json = ""
+
+        while step_count < args.max_steps:
+            step_count += 1
+            steps_executed = step_count
+            log.info(f"\n--- 🔄 Step {step_count} ---")
+            reporter.emit_event("step_started", step=step_count)
+
+            ui_json, screenshot_base64 = _capture_ui_state(
+                args, adapter, reporter, step_count
+            )
+            current_history = history_manager.get_history()
+
+            if last_ui_json == ui_json and step_count > 1 and not last_error:
+                last_error = "[System Warning] The previous action was executed but the page UI did not change. Possible causes: invalid input, unchecked checkbox, or invisible overlay. Do NOT repeat the same action — try a different strategy."
+                log.warning("[E020] UI stagnation detected: action executed but page did not change. Possible causes: invalid input, unchecked checkbox, or invisible overlay.")
+
+            last_ui_json = ui_json
+
+            decision_data = brain.get_next_autonomous_action(
+                goal=args.goal,
+                context=context_content,
+                ui_json=ui_json,
+                history=current_history,
+                platform=args.platform,
+                last_error=last_error,
+                screenshot_base64=screenshot_base64,
+            )
+
+            status = decision_data.get("status")
+            action_data = decision_data.get("result", {})
+            last_error = ""
+            reporter.emit_event(
+                "decision_received",
+                step=step_count,
+                status=status,
+                action=action_data.get("action", ""),
+                locator_type=action_data.get("locator_type", ""),
+                locator_value=action_data.get("locator_value", ""),
+            )
+
+            if status == "success":
+                if action_data and action_data.get("action"):
+                    log.info(
+                        "🔍 Final action/assertion detected with success status, executing..."
+                    )
+                    result = executor.execute_and_record(action_data)
+                    if result.get("success"):
+                        history_manager.add_step(
+                            result["code_lines"], result["action_description"]
+                        )
+                        save_to_disk(
+                            output_script_path,
+                            history_manager.get_current_file_content(),
+                        )
+                        log.info(f"✅ Final action succeeded: {result['action_description']}")
+                        reporter.emit_event(
+                            "action_executed",
+                            step=step_count,
+                            success=True,
+                            action_description=result["action_description"],
+                        )
+                    else:
+                        final_error = "Final action/assertion failed — task verification did not pass"
+                        reporter.emit_event(
+                            "action_executed",
+                            step=step_count,
+                            success=False,
+                            action_description="final_action_failed",
+                        )
+                        log.error("❌ Final action/assertion failed — task verification did not pass")
+                        return 1
+
+                final_status = "success"
+                exit_code = 0
+                log.info("🎉 [Agent] All goals and assertions achieved!")
+                return 0
+
+            if status == "failed":
+                final_error = "Task cannot continue — AI determined failure"
+                log.warning("⚠️ [Agent] Task cannot continue — AI determined failure.")
+                return 1
+
+            if status == "running":
+                if not action_data:
+                    last_error = "Model returned 'running' status but provided no action. Please provide a concrete action."
+                    consecutive_failures += 1
+                    reporter.emit_event(
+                        "action_executed",
+                        step=step_count,
+                        success=False,
+                        action_description="missing_action",
+                    )
+                elif action_data.get("action") == "not_found":
+                    # The model explicitly reports the target is absent from the
+                    # UI tree and needs visual verification. This is not an
+                    # engine error and must NOT count toward the circuit breaker
+                    # (otherwise two not_found in a row trips it before vision is
+                    # ever tried, since max_retries defaults to 2).
+                    last_error = (
+                        "Model reported the target element is not in the current "
+                        "UI tree (action='not_found'). Re-run with --vision to let "
+                        "the visual model locate it, or try a different strategy/"
+                        "scroll to bring it into view."
+                    )
+                    log.warning(
+                        "[Agent] Model returned 'not_found' — target absent from UI "
+                        "tree. Suggesting visual fallback (no circuit-breaker penalty)."
+                    )
+                    reporter.emit_event(
+                        "action_executed",
+                        step=step_count,
+                        success=False,
+                        action_description="not_found",
+                    )
+                    continue
+                else:
+                    result = executor.execute_and_record(action_data)
+                    if result.get("success"):
+                        consecutive_failures = 0
+                        history_manager.add_step(
+                            result["code_lines"], result["action_description"]
+                        )
+                        save_to_disk(
+                            output_script_path,
+                            history_manager.get_current_file_content(),
+                        )
+                        log.info(f"Action succeeded: {result['action_description']}")
+                        reporter.emit_event(
+                            "action_executed",
+                            step=step_count,
+                            success=True,
+                            action_description=result["action_description"],
+                        )
+                    else:
+                        consecutive_failures += 1
+                        action_repr = f"{action_data.get('action')} - {action_data.get('locator_type')}={action_data.get('locator_value')}"
+                        last_error = f"Action [{action_repr}] failed — element not found or not interactable on the current page."
+                        reporter.emit_event(
+                            "action_executed",
+                            step=step_count,
+                            success=False,
+                            action_description=action_repr,
+                        )
+                        log.warning(
+                            f"[E021] Step failed (attempt {consecutive_failures}/{args.max_retries}). Self-heal engine will retry with adjusted strategy."
+                        )
+
+                if consecutive_failures >= args.max_retries:
+                    action_repr = f"{action_data.get('action')} - {action_data.get('locator_type')}={action_data.get('locator_value')}"
+                    final_error = f"Circuit breaker triggered after {args.max_retries} consecutive failures on: {action_repr}"
+                    log.error(
+                        f"[E022] Circuit breaker triggered: {args.max_retries} consecutive failures on [{action_repr}]. "
+                        f"Last error: element not found or not interactable. "
+                        f"Fix: run 'screenforge --action click --platform {args.platform} --dry-run ...' to inspect current page state, "
+                        f"or adjust your workflow/locator strategy."
+                    )
+                    return 1
+                continue
+
+            final_error = f"Unknown decision status: {status}"
+            log.error(f"[E023] Unknown decision status '{status}' returned by AI brain. This is likely a model parsing issue. Fix: check MODEL_NAME supports structured JSON output")
+            return 1
+
+        final_error = f"Max step limit reached ({args.max_steps} steps)"
+        log.warning(
+            f"[E024] Exploration exceeded max step limit ({args.max_steps}). Possible infinite loop. Fix: increase --max_steps or refine your --goal to be more specific."
+        )
+        return 1
+
+    except KeyboardInterrupt:
+        final_error = "Interrupted by user (KeyboardInterrupt)"
+        reporter.emit_event("interrupted", reason="KeyboardInterrupt")
+        log.warning("\n⚠️ Interrupted — safely aborting...")
+        return 1
+
+    finally:
+        reporter.finalize(
+            status=final_status,
+            exit_code=exit_code,
+            steps_executed=steps_executed,
+            last_error=final_error,
+        )
+        log.info(f"🏁 Done. Generated script saved to: {output_script_path}")
+        if adapter:
+            try:
+                adapter.teardown()
+            except Exception as e:
+                log.warning(f"⚠️ [Warning] Cleanup failed: {e}")

cli/modes/demo.py

@@ -0,0 +1,162 @@
+"""Demo mode — showcases ScreenForge's full flow without API key or device."""
+
+import sys
+import time
+
+from cli._version import __version__
+
+DEMO_SCENARIO = {
+    "title": "Login Flow Test",
+    "url": "https://demo.screenforge.dev",
+    "steps": [
+        {
+            "description": "Navigate to login page",
+            "action": "goto",
+            "target": "https://demo.screenforge.dev/login",
+            "ui_tree_snippet": (
+                '<div id="app">\n'
+                '  <nav>\n'
+                '    <a href="/login">Sign In</a>\n'
+                '    <a href="/register">Sign Up</a>\n'
+                '  </nav>\n'
+                '</div>'
+            ),
+            "generated_code": 'page.goto("https://demo.screenforge.dev/login")',
+        },
+        {
+            "description": "Enter username",
+            "action": "input",
+            "target": "#email → 'user@example.com'",
+            "ui_tree_snippet": (
+                '<form id="login-form">\n'
+                '  <input id="email" type="email" placeholder="Email" />\n'
+                '  <input id="password" type="password" placeholder="Password" />\n'
+                '  <button type="submit">Sign In</button>\n'
+                '</form>'
+            ),
+            "generated_code": (
+                'page.locator("#email").first.fill("user@example.com")'
+            ),
+        },
+        {
+            "description": "Enter password",
+            "action": "input",
+            "target": "#password → '••••••••'",
+            "ui_tree_snippet": None,
+            "generated_code": 'page.locator("#password").first.fill("password123")',
+        },
+        {
+            "description": "Click sign in button",
+            "action": "click",
+            "target": "text='Sign In'",
+            "ui_tree_snippet": None,
+            "generated_code": "page.get_by_text('Sign In').first.click()",
+        },
+        {
+            "description": "Verify dashboard loaded",
+            "action": "assert_exist",
+            "target": "text='Welcome back'",
+            "ui_tree_snippet": (
+                '<main>\n'
+                '  <h1>Welcome back</h1>\n'
+                '  <div class="dashboard">...</div>\n'
+                '</main>'
+            ),
+            "generated_code": (
+                'expect(page.get_by_text("Welcome back").first).to_be_visible()'
+            ),
+        },
+    ],
+}
+
+PYTEST_OUTPUT = '''import allure
+import pytest
+from playwright.sync_api import Page, expect
+
+
+@allure.feature("Authentication")
+@allure.story("Login Flow")
+class TestLogin:
+    @allure.step("Navigate to login page")
+    def test_login_success(self, page: Page):
+        page.goto("https://demo.screenforge.dev/login")
+
+        with allure.step("Enter credentials"):
+            page.locator("#email").first.fill("user@example.com")
+            page.locator("#password").first.fill("password123")
+
+        with allure.step("Submit login form"):
+            page.get_by_text('Sign In').first.click()
+
+        with allure.step("Verify dashboard loaded"):
+            expect(page.get_by_text("Welcome back").first).to_be_visible()
+'''
+
+
+def _print_slow(text: str, delay: float = 0.02) -> None:
+    for char in text:
+        sys.stdout.write(char)
+        sys.stdout.flush()
+        time.sleep(delay)
+    sys.stdout.write("\n")
+
+
+def _print_step_header(step_num: int, total: int, description: str) -> None:
+    print(f"\n  [{step_num}/{total}] {description}")
+
+
+def run_demo_mode() -> int:
+    scenario = DEMO_SCENARIO
+    steps = scenario["steps"]
+
+    print()
+    print(f"  screenforge v{__version__} — demo mode")
+    print(f"  Scenario: {scenario['title']}")
+    print(f"  Target:   {scenario['url']} (simulated)")
+    print("  " + "-" * 50)
+    time.sleep(0.5)
+
+    print("\n  Connecting to browser...", end="")
+    time.sleep(0.3)
+    print(" OK (simulated)")
+
+    for i, step in enumerate(steps, 1):
+        _print_step_header(i, len(steps), step["description"])
+        time.sleep(0.3)
+
+        if step["ui_tree_snippet"]:
+            print("    inspect_ui:")
+            for line in step["ui_tree_snippet"].split("\n"):
+                print(f"      {line}")
+            time.sleep(0.2)
+
+        print(f"    action: {step['action']} → {step['target']}")
+        time.sleep(0.2)
+
+        print(f"    codegen: {step['generated_code']}")
+        time.sleep(0.3)
+
+    print("\n  " + "-" * 50)
+    print("  Generating pytest script...\n")
+    time.sleep(0.4)
+
+    for line in PYTEST_OUTPUT.strip().split("\n"):
+        _print_slow(f"    {line}", delay=0.008)
+
+    print("\n  " + "-" * 50)
+    print("  Demo complete!")
+    print()
+    print("  What just happened:")
+    print("    1. ScreenForge inspected the page structure (inspect_ui)")
+    print("    2. AI analyzed the DOM and chose actions")
+    print("    3. Actions were executed step by step")
+    print("    4. A production-ready pytest script was generated")
+    print()
+    print("  To do this for real:")
+    print("    export OPENAI_API_KEY=sk-...")
+    print("    screenforge --action click --platform web --locator-type text --locator-value 'Login'")
+    print()
+    print("  Star us on GitHub if this was useful!")
+    print()
+
+    return 0