screenforge 0.6.1__tar.gz → 0.10.0__tar.gz

{screenforge-0.6.1 → screenforge-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: screenforge
-Version: 0.6.1
+Version: 0.10.0
 Summary: AI-driven cross-platform UI automation engine with test script generation
 License: MIT
 Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
@@ -119,6 +119,9 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
 - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
 - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
 - **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
+- **Execution-replay review report**: Run any test with `REVIEW_RECORD=1` and get a self-contained, shareable offline `report.html` — a drag-timeline of each step's screenshot + the test source line that triggered it + the DOM tree, plus a `video.gif` filmstrip. Default off, zero overhead. (Web verified; mobile registry/recording seam in place but unverified.)
+- **Case memory hints on `inspect_ui`**: pass an optional `task` label and the payload's `memory` field carries what the last run of that task did — locator, counts, and outcome — a hint the Agent weighs against the live tree, seen before it acts, never auto-replayed. It records failures too, so check `last_status`/`success_count` before trusting a hint. The label must match the one the earlier run recorded (`--action-name`, else `"{action}:{locator_value}"`); `load_case_memory` lists them. See the [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md).
+- **Plain-language element lookup on `inspect_ui`**: pass an optional `intent` phrase (e.g. `"登录按钮"`) and the payload's `candidates` field returns up to 3 ranked elements with ready-to-use locators — additive to the full `ui_tree`, never a replacement. Matching is literal string similarity, not semantic, so phrase `intent` in the same language as the UI. See the [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md).
 
 ## Agent Integration (Claude Code / Cursor / Codex)
 

{screenforge-0.6.1 → screenforge-0.10.0}/README.md

@@ -74,6 +74,9 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
 - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
 - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
 - **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
+- **Execution-replay review report**: Run any test with `REVIEW_RECORD=1` and get a self-contained, shareable offline `report.html` — a drag-timeline of each step's screenshot + the test source line that triggered it + the DOM tree, plus a `video.gif` filmstrip. Default off, zero overhead. (Web verified; mobile registry/recording seam in place but unverified.)
+- **Case memory hints on `inspect_ui`**: pass an optional `task` label and the payload's `memory` field carries what the last run of that task did — locator, counts, and outcome — a hint the Agent weighs against the live tree, seen before it acts, never auto-replayed. It records failures too, so check `last_status`/`success_count` before trusting a hint. The label must match the one the earlier run recorded (`--action-name`, else `"{action}:{locator_value}"`); `load_case_memory` lists them. See the [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md).
+- **Plain-language element lookup on `inspect_ui`**: pass an optional `intent` phrase (e.g. `"登录按钮"`) and the payload's `candidates` field returns up to 3 ranked elements with ready-to-use locators — additive to the full `ui_tree`, never a replacement. Matching is literal string similarity, not semantic, so phrase `intent` in the same language as the UI. See the [Agent Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/agent_guide.md).
 
 ## Agent Integration (Claude Code / Cursor / Codex)
 

screenforge-0.10.0/cli/_version.py

@@ -0,0 +1 @@
+__version__ = "0.10.0"

{screenforge-0.6.1 → screenforge-0.10.0}/cli/doctor.py

@@ -373,10 +373,6 @@ def _build_doctor_summary(checks: list[dict]) -> dict:
     }
 
 
-def _build_doctor_remediation_items(checks: list[dict]) -> list[str]:
-    return _build_doctor_summary(checks).get("top_items", [])
-
-
 def _build_doctor_check_failure_message(check: dict) -> str:
     details = []
 

{screenforge-0.6.1 → screenforge-0.10.0}/cli/modes/action.py

@@ -26,6 +26,7 @@ from cli.shared import (
     save_to_disk,
 )
 from common.failure_diagnosis import diagnose
+from common.observation import build_observation
 from common.runtime_modes import MODE_RUN
 
 
@@ -69,9 +70,11 @@ def build_failure_payload(
         ui_elements=ui_tree.get("ui_elements", []) or [],
     )
     payload.update(diag.to_dict())
-    payload["ui_tree"] = ui_tree
-    payload["element_count"] = len(ui_tree.get("ui_elements", []) or [])
-    payload["current_url"] = current_url
+    observation = build_observation(ui_tree=ui_tree, current_url=current_url)
+    observation.pop("screenshot_base64")  # engine_error has never carried one
+    observation.pop("memory")             # decision-time hint; action is post-decision
+    observation.pop("candidates")  # engine_error already has its own diagnosis candidates
+    payload.update(observation)
     return payload
 
 
@@ -88,14 +91,16 @@ def build_success_payload(
     Single source of truth shared by the shell --json stdout write and the MCP
     observation stash, so the two can never drift. Pure: no I/O, no device.
     """
+    observation = build_observation(ui_tree=ui_tree, current_url=current_url)
+    observation.pop("screenshot_base64")  # action payloads have never carried one
+    observation.pop("memory")             # decision-time hint; action is post-decision
+    observation.pop("candidates")  # engine_error already has its own diagnosis candidates
     return {
         "ok": True,
         "action": action_name,
         "platform": platform,
-        "ui_tree": ui_tree,
-        "element_count": len(ui_tree.get("ui_elements", []) or []),
+        **observation,
         "output_script": output_script,
-        "current_url": current_url,
     }
 
 

{screenforge-0.6.1 → screenforge-0.10.0}/cli/session.py

@@ -18,10 +18,6 @@ def _session_file(session_id: str) -> str:
     return os.path.join(_SESSION_DIR, f"{session_id}.json")
 
 
-def session_exists(session_id: str) -> bool:
-    return os.path.exists(_session_file(session_id))
-
-
 def load_session(session_id: str) -> dict | None:
     path = _session_file(session_id)
     if not os.path.exists(path):
@@ -125,7 +121,7 @@ def stop_session_recording(session_id: str) -> str:
         pgid = os.getpgid(pid)
         os.killpg(pgid, signal.SIGINT)
         os.waitpid(pid, 0)
-    except (OSError, ChildProcessError):
+    except OSError:
         try:
             os.kill(pid, signal.SIGINT)
             time.sleep(2)

{screenforge-0.6.1 → screenforge-0.10.0}/cli/shared.py

@@ -3,12 +3,6 @@
 import base64
 import time
 
-from common.capabilities import (
-    ACTIONS_REQUIRING_EXTRA_VALUE,
-    GLOBAL_ACTIONS,
-    SUPPORTED_ACTIONS,
-)
-
 
 class _LazyProxy:
     def __init__(self, loader):
@@ -62,9 +56,6 @@ load_workflow_file = None
 WorkflowLoadError = None
 parse_workflow_var_overrides = None
 resolve_workflow_definition = None
-SUPPORTED_INLINE_ACTIONS = SUPPORTED_ACTIONS
-GLOBAL_INLINE_ACTIONS = GLOBAL_ACTIONS
-INLINE_ACTIONS_REQUIRING_EXTRA_VALUE = ACTIONS_REQUIRING_EXTRA_VALUE
 
 
 def _slug_to_test_name(label: str) -> str:

{screenforge-0.6.1 → screenforge-0.10.0}/cli/tool_protocol_handlers.py

@@ -20,6 +20,8 @@ from cli.shared import (
     current_url,
     log,
 )
+from common.failure_diagnosis import rank_candidates
+from common.observation import build_observation
 from common.run_resume import RunContextLoadError, load_run_bundle
 from common.runtime_modes import MODE_DOCTOR, resolve_execution_mode
 from common.tool_protocol import (
@@ -137,6 +139,30 @@ def _find_case_memory_hit(args, execution_mode: str) -> dict | None:
     )
 
 
+def _lookup_task_memory(platform: str, task: str) -> dict:
+    """Case-memory hint for an inspect_ui request, or {} when there's nothing.
+
+    A HINT, not an instruction — see common/observation.py. Matching is fuzzy by
+    design (find_entry ORs control_label with source_ref), so a wrong hit costs
+    one useless hint, never a wrong click. control_kind is "action" because an
+    inspect precedes a single action, and that's the kind those runs record under.
+    Any read failure degrades to {}: a missing hint must never cost the agent
+    its UI tree.
+    """
+    if not str(task).strip():
+        return {}
+    try:
+        entry = _load_case_memory_store().find_entry(
+            platform=platform,
+            control_kind="action",
+            control_label=task,
+        )
+    except Exception as e:
+        log.warning(f"⚠️ [Warning] Case-memory lookup failed, continuing without hint: {e}")
+        return {}
+    return entry or {}
+
+
 def build_load_case_memory_payload(
     platform: str = "",
     control_kind: str = "",
@@ -182,13 +208,6 @@ def build_inspect_ui_payload(request, shared_adapter_manager: _SharedAdapterMana
             1,
         )
 
-        if not screenshot_base64:
-            try:
-                img_bytes = adapter.take_screenshot()
-                screenshot_base64 = base64.b64encode(img_bytes).decode("utf-8")
-            except Exception as e:
-                log.warning(f"⚠️ [Warning] inspect_ui screenshot capture failed: {e}")
-
         try:
             ui_tree = json.loads(ui_json)
         except json.JSONDecodeError:
@@ -207,30 +226,36 @@ def build_inspect_ui_payload(request, shared_adapter_manager: _SharedAdapterMana
             except Exception as e:
                 log.warning(f"⚠️ [Warning] Failed to sync ref cache from inspect_ui: {e}")
 
-        annotated_screenshot_base64 = ""
+        # Annotate in place: ref labels are only meaningful where the compressor
+        # emits ref + bbox + clickable (web). Mobile trees carry none of those, so
+        # annotate_screenshot draws nothing and just re-encodes the image — same
+        # picture, no labels. Either way exactly ONE image ships.
         if screenshot_base64 and ui_tree.get("ui_elements"):
             try:
                 from utils.screenshot_annotator import annotate_screenshot
                 raw_bytes = base64.b64decode(screenshot_base64)
                 annotated_bytes = annotate_screenshot(raw_bytes, ui_tree["ui_elements"])
-                annotated_screenshot_base64 = base64.b64encode(annotated_bytes).decode("utf-8")
+                screenshot_base64 = base64.b64encode(annotated_bytes).decode("utf-8")
             except Exception as e:
                 log.warning(f"⚠️ [Warning] Annotated screenshot generation failed: {e}")
 
-        page_url = current_url(adapter, request.platform)
-
         return {
             "ok": True,
             "operation": "inspect_ui",
             "exit_code": 0,
             "platform": request.platform,
             "env": request.env,
-            "ui_json": ui_json,
-            "ui_tree": ui_tree,
-            "element_count": len(ui_tree.get("ui_elements", []) or []),
-            "screenshot_base64": screenshot_base64 or "",
-            "annotated_screenshot_base64": annotated_screenshot_base64,
-            "current_url": page_url,
+            **build_observation(
+                ui_tree=ui_tree,
+                current_url=current_url(adapter, request.platform),
+                screenshot_base64=screenshot_base64 or "",
+                memory=_lookup_task_memory(request.platform, request.task),
+                candidates=(
+                    rank_candidates(request.intent, ui_tree.get("ui_elements") or [])
+                    if str(request.intent).strip()
+                    else []
+                ),
+            ),
         }
     except Exception as e:
         return {
@@ -241,6 +266,8 @@ def build_inspect_ui_payload(request, shared_adapter_manager: _SharedAdapterMana
             "env": request.env,
             "error": str(e),
             "current_url": "",
+            "memory": {},
+            "candidates": [],
         }
     finally:
         if owns_adapter and adapter:

{screenforge-0.6.1 → screenforge-0.10.0}/common/adapters/base_adapter.py

@@ -17,6 +17,11 @@ class BasePlatformAdapter(ABC):
     def take_screenshot(self) -> bytes:
         pass
 
+    # 录像 seam（诚实边界）：
+    #   - android/ios：start_record/stop_record_and_get_path 落真视频文件（scrcpy / simctl）。
+    #   - web：start_record 是 no-op（CDP attach 无法用 Playwright 录像）；web 的"录像"
+    #     由 review 层用逐操作截图 ffmpeg 拼胶片（review/render.py make_filmstrip）实现，
+    #     不走本 seam。调用方须把返回 "" / None 当作"无视频"，非错误。
     def start_record(self, video_name: str):
         pass
 

{screenforge-0.6.1 → screenforge-0.10.0}/common/ai.py

@@ -9,6 +9,15 @@ from common.logs import log
 from common.progress import ai_status
 
 
+def _strip_json_fences(text: str) -> str:
+    """Unwrap markdown code fences the model may add despite being told not to."""
+    if "```json" in text:
+        return text.split("```json")[1].split("```")[0].strip()
+    if "```" in text:
+        return text.replace("```", "").strip()
+    return text
+
+
 class AIBrain:
     def __init__(self):
         # 实例化文本专属客户端
@@ -251,6 +260,7 @@ class AIBrain:
         """
         封装底层的 LLM 网络调用
         """
+        result_text = ""
         try:
             with ai_status(f"Thinking ({model_name})..."):
                 response = client.chat.completions.create(
@@ -262,16 +272,11 @@ class AIBrain:
                     temperature=0.1,
                 )
 
-            result_text = response.choices[0].message.content.strip()
-
-            if "```json" in result_text:
-                result_text = result_text.split("```json")[1].split("```")[0].strip()
-            elif "```" in result_text:
-                result_text = result_text.replace("```", "").strip()
-
+            result_text = _strip_json_fences(response.choices[0].message.content.strip())
             parsed_json = json.loads(result_text)
             return parsed_json.get("result", {})
 
         except Exception as e:
-            log.error(f"[Error] Model ({model_name}) request or parse failed: {e}")
+            # {} = "retry without cache" per caller contract; log raw text so parse-fail ≠ network-fail.
+            log.error(f"[Error] Model ({model_name}) request or parse failed: {e}\nRaw response: {result_text}")
             return {}

{screenforge-0.6.1 → screenforge-0.10.0}/common/ai_autonomous.py

@@ -1,7 +1,7 @@
 import json
 
 import config.config as config
-from common.ai import AIBrain
+from common.ai import AIBrain, _strip_json_fences
 from common.logs import log
 from common.progress import ai_status
 
@@ -62,14 +62,10 @@ class AutonomousBrain(AIBrain):
             )
 
         if screenshot_base64:
-            active_client = getattr(self, "vision_client", None) or getattr(
-                self, "text_client", None
-            )
+            active_client = self.vision_client or self.text_client
             active_model = config.VISION_MODEL_NAME
         else:
-            active_client = getattr(self, "text_client", None) or getattr(
-                self, "client", None
-            )
+            active_client = self.text_client
             active_model = config.MODEL_NAME
 
         if not active_client:
@@ -92,13 +88,7 @@ class AutonomousBrain(AIBrain):
                     ],
                     temperature=0.1,
                 )
-            result_text = response.choices[0].message.content.strip()
-
-            if "```json" in result_text:
-                result_text = result_text.split("```json")[1].split("```")[0].strip()
-            elif "```" in result_text:
-                result_text = result_text.replace("```", "").strip()
-
+            result_text = _strip_json_fences(response.choices[0].message.content.strip())
             parsed_json = json.loads(result_text)
             parsed_json.setdefault("current_state_summary", "")
             parsed_json.setdefault("planned_steps", [])
@@ -216,14 +206,10 @@ class AutonomousBrain(AIBrain):
             )
 
         if screenshot_base64:
-            active_client = getattr(self, "vision_client", None) or getattr(
-                self, "text_client", None
-            )  # 兼容配置
+            active_client = self.vision_client or self.text_client  # 兼容配置
             active_model = config.VISION_MODEL_NAME
         else:
-            active_client = getattr(self, "text_client", None) or getattr(
-                self, "client", None
-            )
+            active_client = self.text_client
             active_model = config.MODEL_NAME
 
         if not active_client:
@@ -248,13 +234,7 @@ class AutonomousBrain(AIBrain):
                     temperature=0.1,
                 )
 
-            result_text = response.choices[0].message.content.strip()
-
-            if "```json" in result_text:
-                result_text = result_text.split("```json")[1].split("```")[0].strip()
-            elif "```" in result_text:
-                result_text = result_text.replace("```", "").strip()
-
+            result_text = _strip_json_fences(response.choices[0].message.content.strip())
             parsed_json = json.loads(result_text)
 
             thought = parsed_json.get("thought", "无")

{screenforge-0.6.1 → screenforge-0.10.0}/common/cache/cache_stats.py

@@ -48,11 +48,6 @@ class CacheStats:
     def last_cache_date(self) -> Optional[str]:
         return self._last_cache_date
 
-    def increment_query(self) -> None:
-        self._total_queries += 1
-        self._update_last_cache_date()
-        self._save_stats()
-
     def increment_hit(self) -> None:
         self._total_queries += 1
         self._cache_hits += 1

{screenforge-0.6.1 → screenforge-0.10.0}/common/case_memory.py

@@ -24,6 +24,17 @@ def _slugify(value: str) -> str:
     return text.lower() or "memory"
 
 
+# Inline `--action` runs all record this same source_ref: it says "typed on the
+# command line", not "which control". Treating it as an identity made the first
+# inline action on a platform absorb every later one, so a named action could
+# never get its own entry — or be found by its name.
+_NON_IDENTIFYING_SOURCE_REFS = frozenset({"inline://action"})
+
+
+def _identifying_source_ref(source_ref: str) -> str:
+    return "" if source_ref in _NON_IDENTIFYING_SOURCE_REFS else source_ref
+
+
 def _build_memory_id(
     platform: str,
     control_kind: str,
@@ -240,7 +251,7 @@ class CaseMemoryStore:
         normalized_platform = _normalize_text(platform).lower()
         normalized_kind = _normalize_text(control_kind).lower()
         normalized_label = _normalize_text(control_label)
-        normalized_source_ref = _normalize_text(source_ref)
+        normalized_source_ref = _identifying_source_ref(_normalize_text(source_ref))
         document = self.load_document()
 
         for entry in document.entries:
@@ -277,10 +288,11 @@ class CaseMemoryStore:
 
         document = self.load_document()
         existing_entry = None
+        identifying_source_ref = _identifying_source_ref(source_ref)
         for entry in document.entries:
             if entry.platform != platform or entry.control_kind != control_kind:
                 continue
-            if source_ref and entry.source_ref == source_ref:
+            if identifying_source_ref and entry.source_ref == identifying_source_ref:
                 existing_entry = entry
                 break
             if entry.control_label == control_label: