screenforge 0.4.1__tar.gz → 0.5.0__tar.gz

{screenforge-0.4.1 → screenforge-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: screenforge
-Version: 0.4.1
+Version: 0.5.0
 Summary: AI-driven cross-platform UI automation engine with test script generation
 License: MIT
 Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
@@ -113,6 +113,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
 - **Visual fallback**: When DOM can't locate elements (Canvas, games), VLM parses screenshots
 - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
 - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
+- **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
 
 ## Agent Integration (Claude Code / Cursor / Codex)
 
@@ -205,6 +206,7 @@ If ScreenForge generates tests for your project, add this badge to your README:
 | [MCP Setup (3 min)](https://github.com/jhinzzz/ScreenForge/blob/main/docs/mcp-setup.md) | Connect to Claude Desktop / Cursor / Cline / Claude Code |
 | [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md) | Integration protocol for AI Agents |
 | [Capability Matrix](https://github.com/jhinzzz/ScreenForge/blob/main/docs/capability-matrix.md) | Supported platforms, actions, and locators |
+| [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md) | Live Mirror — watch code + screenshots grow as the test runs |
 | [Workflow Examples](https://github.com/jhinzzz/ScreenForge/tree/main/docs/workflows) | YAML workflow templates |
 | [CHANGELOG](https://github.com/jhinzzz/ScreenForge/blob/main/CHANGELOG.md) | Version history |
 

{screenforge-0.4.1 → screenforge-0.5.0}/README.md

@@ -68,6 +68,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
 - **Visual fallback**: When DOM can't locate elements (Canvas, games), VLM parses screenshots
 - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
 - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
+- **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
 
 ## Agent Integration (Claude Code / Cursor / Codex)
 
@@ -160,6 +161,7 @@ If ScreenForge generates tests for your project, add this badge to your README:
 | [MCP Setup (3 min)](https://github.com/jhinzzz/ScreenForge/blob/main/docs/mcp-setup.md) | Connect to Claude Desktop / Cursor / Cline / Claude Code |
 | [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md) | Integration protocol for AI Agents |
 | [Capability Matrix](https://github.com/jhinzzz/ScreenForge/blob/main/docs/capability-matrix.md) | Supported platforms, actions, and locators |
+| [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md) | Live Mirror — watch code + screenshots grow as the test runs |
 | [Workflow Examples](https://github.com/jhinzzz/ScreenForge/tree/main/docs/workflows) | YAML workflow templates |
 | [CHANGELOG](https://github.com/jhinzzz/ScreenForge/blob/main/CHANGELOG.md) | Version history |
 

screenforge-0.5.0/cli/_version.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{screenforge-0.4.1 → screenforge-0.5.0}/cli/modes/action.py

@@ -5,6 +5,7 @@ import os
 import sys
 
 import cli.shared as _shared
+from cli.playground_sink import build_sink_from_args, maybe_push_step
 from cli.reporter import (
     _apply_resume_summary,
     _build_action_summary,
@@ -206,6 +207,18 @@ def run_action_default_mode(
 
         history_manager.add_step(result["code_lines"], result["action_description"])
         save_to_disk(output_script_path, history_manager.get_current_file_content())
+        # ★ Live-mirror bypass (opt-in --playground-sink). join_on_exit=True: a bare
+        # --action exits right after, so wait briefly for the last frame to land.
+        maybe_push_step(
+            build_sink_from_args(args, join_on_exit=True),
+            args=args,
+            reporter=reporter,
+            adapter=adapter,
+            action_data=action_data,
+            result=result,
+            step_index=None,  # resolver picks: session counter, or 1 for a bare action
+            file_path=output_script_path,  # normalized to abs path inside build_step_event
+        )
         reporter.emit_event(
             "action_executed",
             step=1,

{screenforge-0.4.1 → screenforge-0.5.0}/cli/modes/workflow.py

@@ -4,6 +4,7 @@ from pathlib import Path
 
 import cli.shared as _shared
 from cli.modes.dry_run import _build_resolution_hint, _preview_action_resolution
+from cli.playground_sink import build_sink_from_args, maybe_push_step
 from cli.reporter import (
     _apply_resume_summary,
     _build_reporter,
@@ -285,6 +286,9 @@ def run_workflow_default_mode(
         history_manager = _shared.StepHistoryManager(initial_content=header)
         save_to_disk(output_script_path, header)
         executor = _shared.UIExecutor(device, platform=args.platform)
+        # Workflow is one process with many steps (the main use case for live mirror).
+        # Build the sink once; join_on_exit=False — the process lives across all steps.
+        sink = build_sink_from_args(args, join_on_exit=False)
 
         executed_steps = 0
         for index, step in enumerate(workflow.steps, start=1):
@@ -315,6 +319,17 @@ def run_workflow_default_mode(
                 result["code_lines"], result["action_description"]
             )
             save_to_disk(output_script_path, history_manager.get_current_file_content())
+            # ★ Live-mirror bypass: push this step with its loop counter as step_index.
+            maybe_push_step(
+                sink,
+                args=args,
+                reporter=reporter,
+                adapter=adapter,
+                action_data=action_data,
+                result=result,
+                step_index=index,
+                file_path=output_script_path,  # normalized to abs path inside build_step_event
+            )
             reporter.emit_event(
                 "action_executed",
                 step=index,

{screenforge-0.4.1 → screenforge-0.5.0}/cli/parser.py

@@ -151,6 +151,17 @@ def build_parser() -> argparse.ArgumentParser:
         default=7860,
         help="Playground server port (default: 7860)",
     )
+    parser.add_argument(
+        "--playground-sink",
+        action="store_true",
+        help="Push each step's code + screenshot to a running playground (opt-in; off = zero cost)",
+    )
+    parser.add_argument(
+        "--playground-url",
+        type=str,
+        default="http://127.0.0.1:7860",
+        help="Playground base URL for --playground-sink (default: http://127.0.0.1:7860)",
+    )
     parser.add_argument(
         "--device-url",
         type=str,
@@ -270,7 +281,10 @@ def validate_cli_args(args: argparse.Namespace) -> None:
         raise ValueError("--action cannot be combined with --goal or --workflow")
     has_demo = bool(getattr(args, "demo", False))
     has_init = bool(getattr(args, "init", False))
-    if has_demo or has_init:
+    # --playground starts a standalone server (dispatch.py handles it after this
+    # validation, exactly like --init/--demo) — it needs no goal/workflow/action.
+    has_playground = bool(getattr(args, "playground", False))
+    if has_demo or has_init or has_playground:
         return
     has_session_end = bool(str(getattr(args, "session_end", "")).strip())
     if has_session_end:

screenforge-0.5.0/cli/playground_sink.py

@@ -0,0 +1,202 @@
+"""Fire-and-forget visualization sink: short-lived action process → resident playground.
+
+Red line (G5): any network error is swallowed silently. A sink push MUST NEVER
+slow down the action or change its exit code — the 0/1 exit code is a contract to
+the agent (see CLAUDE.md). The sink is a bypass observer hung after save_to_disk;
+it does not touch execute_and_record, codegen, or disk persistence.
+"""
+
+import base64
+import os
+import threading
+
+import requests  # already in requirements.txt (requests==2.32.5) — zero new deps
+from loguru import logger as log
+from pydantic import BaseModel, Field
+
+from cli.session import load_session
+
+DEFAULT_PLAYGROUND_URL = "http://127.0.0.1:7860"
+
+# Latency ceiling on the contract-protected single-step --action path (red line #1:
+# "never slow the action down"). A reachable-but-slow playground must not tax the
+# action beyond this documented budget. (connect, read) is split so a hung peer
+# can't stall on connect; _JOIN_TIMEOUT is the hard cap the single-step process
+# waits for the last frame to land before sys.exit — kept ≤ read+ε and well under
+# human-perceptible. Worst added latency on --action ≈ _JOIN_TIMEOUT.
+_POST_TIMEOUT = (0.2, 0.25)  # (connect, read) seconds
+_JOIN_TIMEOUT = 0.3  # seconds; single-step last-frame grace
+
+
+class PlaygroundStepEvent(BaseModel):
+    """One step pushed to the playground. Shape == the frontend SSE `step` contract."""
+
+    run_id: str
+    step_index: int  # ⭐ time-travel seed: data accumulates/replays by this index
+    code_lines: list[str] = Field(default_factory=list)
+    action_description: str = ""
+    action: str = ""
+    locator_type: str = ""
+    locator_value: str = ""
+    extra_value: str = ""
+    success: bool = True
+    screenshot_b64: str = ""  # empty = no screenshot this step (degrade, never crash)
+    file_path: str = ""  # abs path of the generated test file (for "open in IDE")
+
+
+class PlaygroundSink:
+    """Pushes each step to a running playground, best-effort.
+
+    Disabled by default: enabled=False means zero cost — no HTTP, no thread, and
+    (at the call sites) take_screenshot is never even invoked.
+    """
+
+    def __init__(
+        self,
+        base_url: str = DEFAULT_PLAYGROUND_URL,
+        enabled: bool = False,
+        join_on_exit: bool = False,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.enabled = enabled
+        # Single-step --action exits the process right after push_step returns;
+        # join a short window so the daemon thread's last frame can land (§6 单步收尾).
+        self._join_on_exit = join_on_exit
+
+    def push_step(self, event: PlaygroundStepEvent) -> None:
+        """Best-effort push. Hands off to a daemon thread; the caller returns at
+        once (arch#3: never block the action hot path)."""
+        if not self.enabled:
+            return
+        t = threading.Thread(target=self._post, args=(event,), daemon=True)
+        t.start()
+        if self._join_on_exit:
+            # Single-step --action exits right after this returns; wait a bounded
+            # grace (≤ _JOIN_TIMEOUT) for the last frame to land. This is the hard
+            # ceiling on added latency to the contract path — never grow it past a
+            # human-imperceptible budget (HIGH-1 from review: 0.6s was too generous).
+            t.join(timeout=_JOIN_TIMEOUT)
+
+    def _post(self, event: PlaygroundStepEvent) -> None:
+        try:
+            requests.post(
+                f"{self.base_url}/api/step",
+                json=event.model_dump(),
+                timeout=_POST_TIMEOUT,  # (connect, read) split: a hung playground can't stall us
+            )
+        except Exception as e:  # ConnectionError / Timeout / anything — swallow (G5)
+            log.debug(f"[playground-sink] skip (playground unreachable): {e}")
+
+    @staticmethod
+    def encode_screenshot(adapter) -> str:
+        """Cross-platform: take_screenshot() -> bytes → base64. Can't grab → '' (degrade).
+
+        Platform-agnostic on purpose: all three adapters expose take_screenshot()
+        (base_adapter.py:17 abstract), so no per-platform branching is needed here.
+        """
+        try:
+            png = adapter.take_screenshot()
+            return base64.b64encode(png).decode() if png else ""
+        except Exception as e:
+            log.debug(f"[playground-sink] screenshot skip: {e}")
+            return ""
+
+
+def build_step_event(
+    *,
+    run_key: str,
+    step_index: int,
+    action_data: dict,
+    result: dict,
+    screenshot_b64: str,
+    file_path: str = "",
+) -> PlaygroundStepEvent:
+    """MANDATORY single construction point for every step event (code#4).
+
+    All three entry points (action / workflow / main) build events ONLY through
+    here. Adding a field later (e.g. a seed timestamp) is then one edit, not three
+    — preventing the P9-style schema split where one call site silently drifts.
+
+    file_path is normalized to an absolute path HERE (one idiom, one place) rather
+    than at each call site, so it happens inside maybe_push_step's G5 try/except.
+    Empty stays empty — abspath('') would wrongly yield the cwd, and the frontend
+    treats '' as "no openable file" (disables the IDE button), so guard it.
+    """
+    return PlaygroundStepEvent(
+        run_id=run_key,
+        step_index=step_index,
+        code_lines=result.get("code_lines", []) or [],
+        action_description=result.get("action_description", ""),
+        action=action_data.get("action", ""),
+        locator_type=action_data.get("locator_type", ""),
+        locator_value=action_data.get("locator_value", ""),
+        extra_value=action_data.get("extra_value", ""),
+        success=bool(result.get("success", True)),
+        screenshot_b64=screenshot_b64,
+        file_path=os.path.abspath(file_path) if file_path else "",
+    )
+
+
+def build_sink_from_args(args, *, join_on_exit: bool = False) -> "PlaygroundSink":
+    """Construct a sink from parsed CLI args. Absent flags → disabled (zero cost)."""
+    return PlaygroundSink(
+        base_url=getattr(args, "playground_url", "") or DEFAULT_PLAYGROUND_URL,
+        enabled=bool(getattr(args, "playground_sink", False)),
+        join_on_exit=join_on_exit,
+    )
+
+
+def maybe_push_step(
+    sink: "PlaygroundSink",
+    *,
+    args,
+    reporter,
+    adapter,
+    action_data: dict,
+    result: dict,
+    step_index: int | None = None,
+    file_path: str = "",
+) -> None:
+    """The ONE guarded entry point every call site uses (action / workflow / main).
+
+    Disabled-fast: returns before touching the adapter, so take_screenshot is
+    never called and there is zero device I/O or network on the hot path when the
+    sink is off. Wrapped in a blanket try/except as a belt-and-suspenders G5
+    guard — the bypass observer must never break the action it observes.
+    """
+    if not sink.enabled:
+        return
+    try:
+        run_key, resolved_index = resolve_playground_run_key(args, reporter)
+        event = build_step_event(
+            run_key=run_key,
+            step_index=step_index if step_index is not None else resolved_index,
+            action_data=action_data,
+            result=result,
+            screenshot_b64=PlaygroundSink.encode_screenshot(adapter),
+            file_path=file_path,
+        )
+        sink.push_step(event)
+    except Exception as e:  # never let visualization break the observed action
+        log.debug(f"[playground-sink] push skipped: {e}")
+
+
+def resolve_playground_run_key(args, reporter) -> tuple[str, int]:
+    """Return (run_key, step_index) — the cross-process-stable playground timeline key.
+
+    Root cause (arch#1): run_reporter.py mints run_id as `timestamp_uuid`, unique
+    per short-lived process. In agent mode each --action is its own process, so
+    using reporter.run_id directly would shatter a 5-step flow into 5 single-step
+    buckets and the seed's timeline would be born broken.
+
+    --session-id present → use session_id as the key (one session = one timeline);
+      step_index comes from the session's persisted 'steps' counter (cli/session.py),
+      which dispatch.py increments AFTER each successful step, so steps+1 is the
+      1-based index of the step about to be pushed.
+    No session → a bare --action is inherently single-step: reporter.run_id, index 1.
+    """
+    session_id = getattr(args, "session_id", "") or getattr(args, "session_end", "")
+    if session_id:
+        session = load_session(session_id)
+        return session_id, (session.get("steps", 0) + 1 if session else 1)
+    return reporter.run_id, 1

{screenforge-0.4.1 → screenforge-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "screenforge"
-version = "0.4.1"
+version = "0.5.0"
 description = "AI-driven cross-platform UI automation engine with test script generation"
 readme = "README.md"
 requires-python = ">=3.11"

{screenforge-0.4.1 → screenforge-0.5.0}/screenforge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: screenforge
-Version: 0.4.1
+Version: 0.5.0
 Summary: AI-driven cross-platform UI automation engine with test script generation
 License: MIT
 Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
@@ -113,6 +113,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
 - **Visual fallback**: When DOM can't locate elements (Canvas, games), VLM parses screenshots
 - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
 - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
+- **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
 
 ## Agent Integration (Claude Code / Cursor / Codex)
 
@@ -205,6 +206,7 @@ If ScreenForge generates tests for your project, add this badge to your README:
 | [MCP Setup (3 min)](https://github.com/jhinzzz/ScreenForge/blob/main/docs/mcp-setup.md) | Connect to Claude Desktop / Cursor / Cline / Claude Code |
 | [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md) | Integration protocol for AI Agents |
 | [Capability Matrix](https://github.com/jhinzzz/ScreenForge/blob/main/docs/capability-matrix.md) | Supported platforms, actions, and locators |
+| [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md) | Live Mirror — watch code + screenshots grow as the test runs |
 | [Workflow Examples](https://github.com/jhinzzz/ScreenForge/tree/main/docs/workflows) | YAML workflow templates |
 | [CHANGELOG](https://github.com/jhinzzz/ScreenForge/blob/main/CHANGELOG.md) | Version history |
 

{screenforge-0.4.1 → screenforge-0.5.0}/screenforge.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ cli/_version.py
 cli/dispatch.py
 cli/doctor.py
 cli/parser.py
+cli/playground_sink.py
 cli/reporter.py
 cli/session.py
 cli/shared.py
@@ -78,6 +79,9 @@ tests/test_ios_smoke_live.py
 tests/test_mcp_ref_cache.py
 tests/test_ml_optional.py
 tests/test_parser.py
+tests/test_playground_app.py
+tests/test_playground_sink.py
+tests/test_playground_sink_integration.py
 tests/test_run_reporter.py
 tests/test_run_resume.py
 tests/test_runtime_modes.py

{screenforge-0.4.1 → screenforge-0.5.0}/tests/test_parser.py

@@ -144,3 +144,15 @@ class TestValidateCliArgs:
     def test_demo_valid(self):
         args = self._make_args(goal="", demo=True)
         validate_cli_args(args)
+
+    def test_playground_alone_valid(self):
+        # Regression: --playground starts a standalone server (dispatch handles it
+        # AFTER this validation, like --init/--demo) and needs no goal/action.
+        # Before the fix it raised "Must provide --goal/--workflow/--action",
+        # making the live-mirror entry point impossible to launch from the CLI.
+        args = self._make_args(goal="", playground=True)
+        validate_cli_args(args)
+
+    def test_playground_with_port_valid(self):
+        args = self._make_args(goal="", playground=True, playground_port=8000)
+        validate_cli_args(args)