screenforge 0.6.1__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {screenforge-0.6.1 → screenforge-0.7.0}/PKG-INFO +2 -1
  2. {screenforge-0.6.1 → screenforge-0.7.0}/README.md +1 -0
  3. screenforge-0.7.0/cli/_version.py +1 -0
  4. {screenforge-0.6.1 → screenforge-0.7.0}/cli/doctor.py +0 -4
  5. {screenforge-0.6.1 → screenforge-0.7.0}/cli/session.py +1 -5
  6. {screenforge-0.6.1 → screenforge-0.7.0}/cli/shared.py +0 -9
  7. {screenforge-0.6.1 → screenforge-0.7.0}/common/adapters/base_adapter.py +5 -0
  8. {screenforge-0.6.1 → screenforge-0.7.0}/common/ai.py +13 -8
  9. {screenforge-0.6.1 → screenforge-0.7.0}/common/ai_autonomous.py +7 -27
  10. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/cache_stats.py +0 -5
  11. {screenforge-0.6.1 → screenforge-0.7.0}/common/executor.py +134 -125
  12. {screenforge-0.6.1 → screenforge-0.7.0}/config/config.py +2 -4
  13. screenforge-0.7.0/config/env_loader.py +23 -0
  14. {screenforge-0.6.1 → screenforge-0.7.0}/pyproject.toml +10 -1
  15. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/PKG-INFO +2 -1
  16. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/SOURCES.txt +11 -0
  17. screenforge-0.7.0/tests/test_cache_hash.py +52 -0
  18. screenforge-0.7.0/tests/test_review_cases.py +98 -0
  19. screenforge-0.7.0/tests/test_review_conftest_gating.py +24 -0
  20. screenforge-0.7.0/tests/test_review_filmstrip.py +46 -0
  21. screenforge-0.7.0/tests/test_review_html.py +73 -0
  22. screenforge-0.7.0/tests/test_review_json.py +26 -0
  23. screenforge-0.7.0/tests/test_review_locate_frame.py +32 -0
  24. screenforge-0.7.0/tests/test_review_patching.py +87 -0
  25. screenforge-0.7.0/tests/test_review_recorder.py +66 -0
  26. screenforge-0.7.0/tests/test_review_screenshot.py +41 -0
  27. screenforge-0.7.0/tests/test_review_smoke_live.py +89 -0
  28. screenforge-0.6.1/cli/_version.py +0 -1
  29. screenforge-0.6.1/config/env_loader.py +0 -76
  30. {screenforge-0.6.1 → screenforge-0.7.0}/LICENSE +0 -0
  31. {screenforge-0.6.1 → screenforge-0.7.0}/cli/__init__.py +0 -0
  32. {screenforge-0.6.1 → screenforge-0.7.0}/cli/dispatch.py +0 -0
  33. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/__init__.py +0 -0
  34. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/action.py +0 -0
  35. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/default.py +0 -0
  36. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/demo.py +0 -0
  37. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/dry_run.py +0 -0
  38. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/init.py +0 -0
  39. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/plan.py +0 -0
  40. {screenforge-0.6.1 → screenforge-0.7.0}/cli/modes/workflow.py +0 -0
  41. {screenforge-0.6.1 → screenforge-0.7.0}/cli/parser.py +0 -0
  42. {screenforge-0.6.1 → screenforge-0.7.0}/cli/playground_sink.py +0 -0
  43. {screenforge-0.6.1 → screenforge-0.7.0}/cli/reporter.py +0 -0
  44. {screenforge-0.6.1 → screenforge-0.7.0}/cli/shorthand.py +0 -0
  45. {screenforge-0.6.1 → screenforge-0.7.0}/cli/tool_protocol_handlers.py +0 -0
  46. {screenforge-0.6.1 → screenforge-0.7.0}/common/__init__.py +0 -0
  47. {screenforge-0.6.1 → screenforge-0.7.0}/common/adapters/__init__.py +0 -0
  48. {screenforge-0.6.1 → screenforge-0.7.0}/common/adapters/android_adapter.py +0 -0
  49. {screenforge-0.6.1 → screenforge-0.7.0}/common/adapters/ios_adapter.py +0 -0
  50. {screenforge-0.6.1 → screenforge-0.7.0}/common/adapters/web_adapter.py +0 -0
  51. {screenforge-0.6.1 → screenforge-0.7.0}/common/ai_heal.py +0 -0
  52. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/__init__.py +0 -0
  53. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/cache_hash.py +0 -0
  54. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/cache_manager.py +0 -0
  55. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/cache_storage.py +0 -0
  56. {screenforge-0.6.1 → screenforge-0.7.0}/common/cache/embedding_loader.py +0 -0
  57. {screenforge-0.6.1 → screenforge-0.7.0}/common/capabilities.py +0 -0
  58. {screenforge-0.6.1 → screenforge-0.7.0}/common/case_memory.py +0 -0
  59. {screenforge-0.6.1 → screenforge-0.7.0}/common/error_codes.py +0 -0
  60. {screenforge-0.6.1 → screenforge-0.7.0}/common/exceptions.py +0 -0
  61. {screenforge-0.6.1 → screenforge-0.7.0}/common/failure_diagnosis.py +0 -0
  62. {screenforge-0.6.1 → screenforge-0.7.0}/common/history_manager.py +0 -0
  63. {screenforge-0.6.1 → screenforge-0.7.0}/common/logs.py +0 -0
  64. {screenforge-0.6.1 → screenforge-0.7.0}/common/mcp_server.py +0 -0
  65. {screenforge-0.6.1 → screenforge-0.7.0}/common/preflight.py +0 -0
  66. {screenforge-0.6.1 → screenforge-0.7.0}/common/progress.py +0 -0
  67. {screenforge-0.6.1 → screenforge-0.7.0}/common/run_reporter.py +0 -0
  68. {screenforge-0.6.1 → screenforge-0.7.0}/common/run_resume.py +0 -0
  69. {screenforge-0.6.1 → screenforge-0.7.0}/common/runtime_modes.py +0 -0
  70. {screenforge-0.6.1 → screenforge-0.7.0}/common/tool_protocol.py +0 -0
  71. {screenforge-0.6.1 → screenforge-0.7.0}/common/visual_fallback.py +0 -0
  72. {screenforge-0.6.1 → screenforge-0.7.0}/common/workflow_schema.py +0 -0
  73. {screenforge-0.6.1 → screenforge-0.7.0}/config/__init__.py +0 -0
  74. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/dependency_links.txt +0 -0
  75. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/entry_points.txt +0 -0
  76. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/requires.txt +0 -0
  77. {screenforge-0.6.1 → screenforge-0.7.0}/screenforge.egg-info/top_level.txt +0 -0
  78. {screenforge-0.6.1 → screenforge-0.7.0}/setup.cfg +0 -0
  79. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_action_mode_observation_stash.py +0 -0
  80. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_ai_autonomous.py +0 -0
  81. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_ai_brain.py +0 -0
  82. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_ai_heal.py +0 -0
  83. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_android_smoke_live.py +0 -0
  84. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_cache_manager.py +0 -0
  85. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_capabilities.py +0 -0
  86. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_cli_action_json.py +0 -0
  87. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_codegen_quality.py +0 -0
  88. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_dispatch.py +0 -0
  89. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_doctor_orphan_browser.py +0 -0
  90. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_dom_capture.py +0 -0
  91. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_error_codes.py +0 -0
  92. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_executor.py +0 -0
  93. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_failure_diagnosis.py +0 -0
  94. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_interaction_actions.py +0 -0
  95. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_ios_smoke_live.py +0 -0
  96. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_mcp_execute_observation.py +0 -0
  97. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_mcp_execute_parity_live.py +0 -0
  98. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_mcp_ref_cache.py +0 -0
  99. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_ml_optional.py +0 -0
  100. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_mode_error_code_threading.py +0 -0
  101. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_parser.py +0 -0
  102. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_playground_app.py +0 -0
  103. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_playground_sink.py +0 -0
  104. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_playground_sink_integration.py +0 -0
  105. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_run_reporter.py +0 -0
  106. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_run_resume.py +0 -0
  107. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_runtime_modes.py +0 -0
  108. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_screenshot_annotator.py +0 -0
  109. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_shared_observation_stash.py +0 -0
  110. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_shorthand.py +0 -0
  111. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_tool_protocol_diagnosis.py +0 -0
  112. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_utils_ios.py +0 -0
  113. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_utils_web.py +0 -0
  114. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_utils_xml.py +0 -0
  115. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_visual_fallback.py +0 -0
  116. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_web_adapter.py +0 -0
  117. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_web_dom_complex_live.py +0 -0
  118. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_web_smoke_live.py +0 -0
  119. {screenforge-0.6.1 → screenforge-0.7.0}/tests/test_workflow_mode_observation_stash.py +0 -0
  120. {screenforge-0.6.1 → screenforge-0.7.0}/utils/__init__.py +0 -0
  121. {screenforge-0.6.1 → screenforge-0.7.0}/utils/screenshot_annotator.py +0 -0
  122. {screenforge-0.6.1 → screenforge-0.7.0}/utils/utils_ios.py +0 -0
  123. {screenforge-0.6.1 → screenforge-0.7.0}/utils/utils_web.py +0 -0
  124. {screenforge-0.6.1 → screenforge-0.7.0}/utils/utils_xml.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: screenforge
3
- Version: 0.6.1
3
+ Version: 0.7.0
4
4
  Summary: AI-driven cross-platform UI automation engine with test script generation
5
5
  License: MIT
6
6
  Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
@@ -119,6 +119,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
119
119
  - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
120
120
  - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
121
121
  - **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
122
+ - **Execution-replay review report**: Run any test with `REVIEW_RECORD=1` and get a self-contained, shareable offline `report.html` — a drag-timeline of each step's screenshot + the test source line that triggered it + the DOM tree, plus a `video.gif` filmstrip. Default off, zero overhead. (Web verified; mobile registry/recording seam in place but unverified.)
122
123
 
123
124
  ## Agent Integration (Claude Code / Cursor / Codex)
124
125
 
@@ -74,6 +74,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
74
74
  - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
75
75
  - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
76
76
  - **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
77
+ - **Execution-replay review report**: Run any test with `REVIEW_RECORD=1` and get a self-contained, shareable offline `report.html` — a drag-timeline of each step's screenshot + the test source line that triggered it + the DOM tree, plus a `video.gif` filmstrip. Default off, zero overhead. (Web verified; mobile registry/recording seam in place but unverified.)
77
78
 
78
79
  ## Agent Integration (Claude Code / Cursor / Codex)
79
80
 
@@ -0,0 +1 @@
1
+ __version__ = "0.7.0"
@@ -373,10 +373,6 @@ def _build_doctor_summary(checks: list[dict]) -> dict:
373
373
  }
374
374
 
375
375
 
376
- def _build_doctor_remediation_items(checks: list[dict]) -> list[str]:
377
- return _build_doctor_summary(checks).get("top_items", [])
378
-
379
-
380
376
  def _build_doctor_check_failure_message(check: dict) -> str:
381
377
  details = []
382
378
 
@@ -18,10 +18,6 @@ def _session_file(session_id: str) -> str:
18
18
  return os.path.join(_SESSION_DIR, f"{session_id}.json")
19
19
 
20
20
 
21
- def session_exists(session_id: str) -> bool:
22
- return os.path.exists(_session_file(session_id))
23
-
24
-
25
21
  def load_session(session_id: str) -> dict | None:
26
22
  path = _session_file(session_id)
27
23
  if not os.path.exists(path):
@@ -125,7 +121,7 @@ def stop_session_recording(session_id: str) -> str:
125
121
  pgid = os.getpgid(pid)
126
122
  os.killpg(pgid, signal.SIGINT)
127
123
  os.waitpid(pid, 0)
128
- except (OSError, ChildProcessError):
124
+ except OSError:
129
125
  try:
130
126
  os.kill(pid, signal.SIGINT)
131
127
  time.sleep(2)
@@ -3,12 +3,6 @@
3
3
  import base64
4
4
  import time
5
5
 
6
- from common.capabilities import (
7
- ACTIONS_REQUIRING_EXTRA_VALUE,
8
- GLOBAL_ACTIONS,
9
- SUPPORTED_ACTIONS,
10
- )
11
-
12
6
 
13
7
  class _LazyProxy:
14
8
  def __init__(self, loader):
@@ -62,9 +56,6 @@ load_workflow_file = None
62
56
  WorkflowLoadError = None
63
57
  parse_workflow_var_overrides = None
64
58
  resolve_workflow_definition = None
65
- SUPPORTED_INLINE_ACTIONS = SUPPORTED_ACTIONS
66
- GLOBAL_INLINE_ACTIONS = GLOBAL_ACTIONS
67
- INLINE_ACTIONS_REQUIRING_EXTRA_VALUE = ACTIONS_REQUIRING_EXTRA_VALUE
68
59
 
69
60
 
70
61
  def _slug_to_test_name(label: str) -> str:
@@ -17,6 +17,11 @@ class BasePlatformAdapter(ABC):
17
17
  def take_screenshot(self) -> bytes:
18
18
  pass
19
19
 
20
+ # 录像 seam(诚实边界):
21
+ # - android/ios:start_record/stop_record_and_get_path 落真视频文件(scrcpy / simctl)。
22
+ # - web:start_record 是 no-op(CDP attach 无法用 Playwright 录像);web 的"录像"
23
+ # 由 review 层用逐操作截图 ffmpeg 拼胶片(review/render.py make_filmstrip)实现,
24
+ # 不走本 seam。调用方须把返回 "" / None 当作"无视频",非错误。
20
25
  def start_record(self, video_name: str):
21
26
  pass
22
27
 
@@ -9,6 +9,15 @@ from common.logs import log
9
9
  from common.progress import ai_status
10
10
 
11
11
 
12
+ def _strip_json_fences(text: str) -> str:
13
+ """Unwrap markdown code fences the model may add despite being told not to."""
14
+ if "```json" in text:
15
+ return text.split("```json")[1].split("```")[0].strip()
16
+ if "```" in text:
17
+ return text.replace("```", "").strip()
18
+ return text
19
+
20
+
12
21
  class AIBrain:
13
22
  def __init__(self):
14
23
  # 实例化文本专属客户端
@@ -251,6 +260,7 @@ class AIBrain:
251
260
  """
252
261
  封装底层的 LLM 网络调用
253
262
  """
263
+ result_text = ""
254
264
  try:
255
265
  with ai_status(f"Thinking ({model_name})..."):
256
266
  response = client.chat.completions.create(
@@ -262,16 +272,11 @@ class AIBrain:
262
272
  temperature=0.1,
263
273
  )
264
274
 
265
- result_text = response.choices[0].message.content.strip()
266
-
267
- if "```json" in result_text:
268
- result_text = result_text.split("```json")[1].split("```")[0].strip()
269
- elif "```" in result_text:
270
- result_text = result_text.replace("```", "").strip()
271
-
275
+ result_text = _strip_json_fences(response.choices[0].message.content.strip())
272
276
  parsed_json = json.loads(result_text)
273
277
  return parsed_json.get("result", {})
274
278
 
275
279
  except Exception as e:
276
- log.error(f"[Error] Model ({model_name}) request or parse failed: {e}")
280
+ # {} = "retry without cache" per caller contract; log raw text so parse-fail network-fail.
281
+ log.error(f"[Error] Model ({model_name}) request or parse failed: {e}\nRaw response: {result_text}")
277
282
  return {}
@@ -1,7 +1,7 @@
1
1
  import json
2
2
 
3
3
  import config.config as config
4
- from common.ai import AIBrain
4
+ from common.ai import AIBrain, _strip_json_fences
5
5
  from common.logs import log
6
6
  from common.progress import ai_status
7
7
 
@@ -62,14 +62,10 @@ class AutonomousBrain(AIBrain):
62
62
  )
63
63
 
64
64
  if screenshot_base64:
65
- active_client = getattr(self, "vision_client", None) or getattr(
66
- self, "text_client", None
67
- )
65
+ active_client = self.vision_client or self.text_client
68
66
  active_model = config.VISION_MODEL_NAME
69
67
  else:
70
- active_client = getattr(self, "text_client", None) or getattr(
71
- self, "client", None
72
- )
68
+ active_client = self.text_client
73
69
  active_model = config.MODEL_NAME
74
70
 
75
71
  if not active_client:
@@ -92,13 +88,7 @@ class AutonomousBrain(AIBrain):
92
88
  ],
93
89
  temperature=0.1,
94
90
  )
95
- result_text = response.choices[0].message.content.strip()
96
-
97
- if "```json" in result_text:
98
- result_text = result_text.split("```json")[1].split("```")[0].strip()
99
- elif "```" in result_text:
100
- result_text = result_text.replace("```", "").strip()
101
-
91
+ result_text = _strip_json_fences(response.choices[0].message.content.strip())
102
92
  parsed_json = json.loads(result_text)
103
93
  parsed_json.setdefault("current_state_summary", "")
104
94
  parsed_json.setdefault("planned_steps", [])
@@ -216,14 +206,10 @@ class AutonomousBrain(AIBrain):
216
206
  )
217
207
 
218
208
  if screenshot_base64:
219
- active_client = getattr(self, "vision_client", None) or getattr(
220
- self, "text_client", None
221
- ) # 兼容配置
209
+ active_client = self.vision_client or self.text_client # 兼容配置
222
210
  active_model = config.VISION_MODEL_NAME
223
211
  else:
224
- active_client = getattr(self, "text_client", None) or getattr(
225
- self, "client", None
226
- )
212
+ active_client = self.text_client
227
213
  active_model = config.MODEL_NAME
228
214
 
229
215
  if not active_client:
@@ -248,13 +234,7 @@ class AutonomousBrain(AIBrain):
248
234
  temperature=0.1,
249
235
  )
250
236
 
251
- result_text = response.choices[0].message.content.strip()
252
-
253
- if "```json" in result_text:
254
- result_text = result_text.split("```json")[1].split("```")[0].strip()
255
- elif "```" in result_text:
256
- result_text = result_text.replace("```", "").strip()
257
-
237
+ result_text = _strip_json_fences(response.choices[0].message.content.strip())
258
238
  parsed_json = json.loads(result_text)
259
239
 
260
240
  thought = parsed_json.get("thought", "无")
@@ -48,11 +48,6 @@ class CacheStats:
48
48
  def last_cache_date(self) -> Optional[str]:
49
49
  return self._last_cache_date
50
50
 
51
- def increment_query(self) -> None:
52
- self._total_queries += 1
53
- self._update_last_cache_date()
54
- self._save_stats()
55
-
56
51
  def increment_hit(self) -> None:
57
52
  self._total_queries += 1
58
53
  self._cache_hits += 1
@@ -1213,8 +1213,6 @@ class DragHandler(ActionHandler):
1213
1213
 
1214
1214
 
1215
1215
  class UIExecutor:
1216
- _handlers = {}
1217
-
1218
1216
  def __init__(self, device, platform="android"):
1219
1217
  self.d = device
1220
1218
  self.platform = platform
@@ -1223,33 +1221,28 @@ class UIExecutor:
1223
1221
  # and a follow-up `ref @N` action in the same MCP session share this
1224
1222
  # cache while separate sessions/pages can never leak into each other.
1225
1223
  self._cached_ui_elements: list[dict] = []
1226
- if not self._handlers:
1227
- self._handlers = {
1228
- "click": ClickHandler(),
1229
- "long_click": LongClickHandler(),
1230
- "hover": HoverHandler(),
1231
- "input": InputHandler(),
1232
- "swipe": SwipeHandler(),
1233
- "press": PressHandler(),
1234
- "goto": GotoHandler(),
1235
- "scroll_into_view": ScrollIntoViewHandler(),
1236
- "select": SelectHandler(),
1237
- "upload": UploadHandler(),
1238
- "double_click": DoubleClickHandler(),
1239
- "right_click": RightClickHandler(),
1240
- "drag": DragHandler(),
1241
- "wait_for": WaitForHandler(),
1242
- "assert_exist": AssertExistHandler(),
1243
- "assert_not_exist": AssertNotExistHandler(),
1244
- "assert_text_equals": AssertTextEqualsHandler(),
1245
- "assert_text_contains": AssertTextContainsHandler(),
1246
- "assert_value": AssertValueHandler(),
1247
- "assert_url": AssertUrlHandler(),
1248
- }
1249
-
1250
- @classmethod
1251
- def register_handler(cls, action_type: str, handler: ActionHandler):
1252
- cls._handlers[action_type] = handler
1224
+ self._handlers = {
1225
+ "click": ClickHandler(),
1226
+ "long_click": LongClickHandler(),
1227
+ "hover": HoverHandler(),
1228
+ "input": InputHandler(),
1229
+ "swipe": SwipeHandler(),
1230
+ "press": PressHandler(),
1231
+ "goto": GotoHandler(),
1232
+ "scroll_into_view": ScrollIntoViewHandler(),
1233
+ "select": SelectHandler(),
1234
+ "upload": UploadHandler(),
1235
+ "double_click": DoubleClickHandler(),
1236
+ "right_click": RightClickHandler(),
1237
+ "drag": DragHandler(),
1238
+ "wait_for": WaitForHandler(),
1239
+ "assert_exist": AssertExistHandler(),
1240
+ "assert_not_exist": AssertNotExistHandler(),
1241
+ "assert_text_equals": AssertTextEqualsHandler(),
1242
+ "assert_text_contains": AssertTextContainsHandler(),
1243
+ "assert_value": AssertValueHandler(),
1244
+ "assert_url": AssertUrlHandler(),
1245
+ }
1253
1246
 
1254
1247
  def set_ui_elements(self, elements: list[dict]) -> None:
1255
1248
  self._cached_ui_elements = list(elements) if elements else []
@@ -1260,6 +1253,114 @@ class UIExecutor:
1260
1253
  return el
1261
1254
  return None
1262
1255
 
1256
+ def _recover_web_ref(self, handler, action, l_value, extra_value, result: dict) -> dict:
1257
+ """Web @N ref that get_actual_element couldn't resolve. Try, in order:
1258
+ a recovered stable locator (re-driven through the action's own handler),
1259
+ then a live coordinate click for `click` only (persisted as a skip, never
1260
+ a rotting coordinate). Always terminal — returns the result dict."""
1261
+ el_data = self.resolve_ref(l_value)
1262
+ # The id→text chain in get_actual_element missed this ref, but the
1263
+ # element dict may still carry name/role/desc/placeholder. Recover
1264
+ # the SAME stable locator we'd emit (build_fallback_locator) as a
1265
+ # LIVE handle and re-drive the action's own handler on it — so an
1266
+ # input stays an input, a hover stays a hover, etc. The handler's
1267
+ # execute() validates the locator resolves (discharges the "locator
1268
+ # points at a different element than the pixel" trap) and its
1269
+ # generate_code() persists that stable locator — never a pixel.
1270
+ fb_element = get_fallback_element(self.d, el_data) if el_data else None
1271
+ if fb_element is not None and build_fallback_locator(el_data):
1272
+ try:
1273
+ if handler.execute(self.d, fb_element, self.platform, extra_value):
1274
+ # generate_code with u2_key="ref" + the live resolve_ref
1275
+ # rebuilds the SAME stable fragment build_fallback_locator
1276
+ # produced (build_code's ref branch calls it) — so the
1277
+ # persisted locator exactly matches the handle we just
1278
+ # acted on. Then humanize the [@N] labels to readable.
1279
+ code_lines = handler.generate_code(
1280
+ self.platform, "ref", l_value, extra_value,
1281
+ config.DEFAULT_TIMEOUT, resolve_ref=self.resolve_ref,
1282
+ )
1283
+ readable = readable_ref_target(el_data)
1284
+ if readable and readable != l_value:
1285
+ code_lines = humanize_step_labels(code_lines, l_value, readable)
1286
+ result["success"] = True
1287
+ result["code_lines"] = code_lines
1288
+ result["action_description"] = (
1289
+ f"🔁 [Ref] {action} {l_value} via recovered locator"
1290
+ )
1291
+ return result
1292
+ log.warning(f"⚠️ [Ref] Recovered locator did not satisfy {action}")
1293
+ except Exception as e:
1294
+ log.warning(f"⚠️ [Ref] Recovered locator failed to act: {e}")
1295
+ # No stable locator. Act LIVE via coordinate ONLY for click (the
1296
+ # sole action a bare coordinate can perform) so a recording session
1297
+ # still advances; persist an honest, non-passing skip rather than a
1298
+ # silently-rotting coordinate. For non-click actions there is
1299
+ # nothing a coordinate can do — report a real engine failure.
1300
+ if action == "click" and el_data and el_data.get("w", 0) > 0:
1301
+ cx = el_data["x"] + el_data["w"] // 2
1302
+ cy = el_data["y"] + el_data["h"] // 2
1303
+ try:
1304
+ self.d.mouse.click(cx, cy)
1305
+ log.info(f"🎯 [Ref] {l_value} clicked at ({cx}, {cy}) live (not persisted)")
1306
+ except Exception as e:
1307
+ log.error(f"❌ [Error] Coordinate click failed: {e}")
1308
+ return result
1309
+ log.warning(
1310
+ f"[E036] Ref {l_value} has no stable locator (only coordinates). "
1311
+ f"Emitting pytest.skip so the test isn't silently green."
1312
+ )
1313
+ code_lines = [
1314
+ f" with allure.step('Click: [{l_value}] (UNREPLAYABLE)'):\n",
1315
+ " import pytest\n",
1316
+ f" # Ref {l_value} could only be located by coordinate at record time;\n",
1317
+ " # no durable selector exists. Provide one before treating this as real.\n",
1318
+ f" pytest.skip('Ref {l_value} has no durable locator (coordinate-only at record time)')\n",
1319
+ ]
1320
+ result["success"] = True
1321
+ result["code_lines"] = code_lines
1322
+ result["action_description"] = f"⏭️ [Ref] {l_value} unreplayable — skip emitted"
1323
+ return result
1324
+ log.error(format_log("E037") + f" (ref={l_value}, action='{action}')")
1325
+ result["error_code"] = "E037"
1326
+ return result
1327
+
1328
+ def _try_visual_fallback(self, l_type, l_value, result: dict) -> dict | None:
1329
+ """VLM screenshot-coordinate click recovery (web click only). Returns the
1330
+ result dict on a hit (persisted as a skip — a VLM point has no durable
1331
+ locator), or None to fall through to the standard not-found path."""
1332
+ try:
1333
+ from common.visual_fallback import visual_locate
1334
+ screenshot_bytes = self.d.screenshot()
1335
+ coords = visual_locate(
1336
+ screenshot_bytes,
1337
+ f"{l_type}={l_value}",
1338
+ )
1339
+ if coords:
1340
+ cx, cy = coords
1341
+ log.info(f"👁️ [Visual Fallback] Using VLM coordinates: ({cx}, {cy})")
1342
+ # Click live so the session advances, but the visual
1343
+ # fallback has NO DOM node — only a VLM screenshot hit —
1344
+ # so there is no durable locator to emit. Persist a skip
1345
+ # (with the coordinate kept as a comment hint) rather than
1346
+ # a coordinate that will rot and pass green against the
1347
+ # wrong pixel later.
1348
+ self.d.mouse.click(cx, cy)
1349
+ code_lines = [
1350
+ f" with allure.step('Click: [{l_value}] (UNREPLAYABLE: visual)'):\n",
1351
+ " import pytest\n",
1352
+ f" # Located only via VLM screenshot at ({cx}, {cy}); no DOM locator exists.\n",
1353
+ f" # d.mouse.click({cx}, {cy}) # original visual hit, reference only\n",
1354
+ f" pytest.skip('Visual-fallback step for [{l_value}] has no durable locator; add a selector to make it replayable')\n",
1355
+ ]
1356
+ result["success"] = True
1357
+ result["code_lines"] = code_lines
1358
+ result["action_description"] = f"⏭️ [Visual Fallback] {l_value} unreplayable — skip emitted"
1359
+ return result
1360
+ except Exception as e:
1361
+ log.warning(f"⚠️ [Visual Fallback] Attempt failed: {e}")
1362
+ return None
1363
+
1263
1364
  def execute_and_record(self, action_data: dict, file_obj=None) -> dict:
1264
1365
  action = action_data.get("action")
1265
1366
  l_type = action_data.get("locator_type", "global")
@@ -1339,106 +1440,14 @@ class UIExecutor:
1339
1440
  return result
1340
1441
 
1341
1442
  if element is None and u2_key == "ref" and self.platform == "web":
1342
- el_data = self.resolve_ref(l_value)
1343
- # The id→text chain in get_actual_element missed this ref, but the
1344
- # element dict may still carry name/role/desc/placeholder. Recover
1345
- # the SAME stable locator we'd emit (build_fallback_locator) as a
1346
- # LIVE handle and re-drive the action's own handler on it — so an
1347
- # input stays an input, a hover stays a hover, etc. The handler's
1348
- # execute() validates the locator resolves (discharges the "locator
1349
- # points at a different element than the pixel" trap) and its
1350
- # generate_code() persists that stable locator — never a pixel.
1351
- fb_element = get_fallback_element(self.d, el_data) if el_data else None
1352
- if fb_element is not None and build_fallback_locator(el_data):
1353
- try:
1354
- if handler.execute(self.d, fb_element, self.platform, extra_value):
1355
- # generate_code with u2_key="ref" + the live resolve_ref
1356
- # rebuilds the SAME stable fragment build_fallback_locator
1357
- # produced (build_code's ref branch calls it) — so the
1358
- # persisted locator exactly matches the handle we just
1359
- # acted on. Then humanize the [@N] labels to readable.
1360
- code_lines = handler.generate_code(
1361
- self.platform, "ref", l_value, extra_value,
1362
- config.DEFAULT_TIMEOUT, resolve_ref=self.resolve_ref,
1363
- )
1364
- readable = readable_ref_target(el_data)
1365
- if readable and readable != l_value:
1366
- code_lines = humanize_step_labels(code_lines, l_value, readable)
1367
- result["success"] = True
1368
- result["code_lines"] = code_lines
1369
- result["action_description"] = (
1370
- f"🔁 [Ref] {action} {l_value} via recovered locator"
1371
- )
1372
- return result
1373
- log.warning(f"⚠️ [Ref] Recovered locator did not satisfy {action}")
1374
- except Exception as e:
1375
- log.warning(f"⚠️ [Ref] Recovered locator failed to act: {e}")
1376
- # No stable locator. Act LIVE via coordinate ONLY for click (the
1377
- # sole action a bare coordinate can perform) so a recording session
1378
- # still advances; persist an honest, non-passing skip rather than a
1379
- # silently-rotting coordinate. For non-click actions there is
1380
- # nothing a coordinate can do — report a real engine failure.
1381
- if action == "click" and el_data and el_data.get("w", 0) > 0:
1382
- cx = el_data["x"] + el_data["w"] // 2
1383
- cy = el_data["y"] + el_data["h"] // 2
1384
- try:
1385
- self.d.mouse.click(cx, cy)
1386
- log.info(f"🎯 [Ref] {l_value} clicked at ({cx}, {cy}) live (not persisted)")
1387
- except Exception as e:
1388
- log.error(f"❌ [Error] Coordinate click failed: {e}")
1389
- return result
1390
- log.warning(
1391
- f"[E036] Ref {l_value} has no stable locator (only coordinates). "
1392
- f"Emitting pytest.skip so the test isn't silently green."
1393
- )
1394
- code_lines = [
1395
- f" with allure.step('Click: [{l_value}] (UNREPLAYABLE)'):\n",
1396
- " import pytest\n",
1397
- f" # Ref {l_value} could only be located by coordinate at record time;\n",
1398
- " # no durable selector exists. Provide one before treating this as real.\n",
1399
- f" pytest.skip('Ref {l_value} has no durable locator (coordinate-only at record time)')\n",
1400
- ]
1401
- result["success"] = True
1402
- result["code_lines"] = code_lines
1403
- result["action_description"] = f"⏭️ [Ref] {l_value} unreplayable — skip emitted"
1404
- return result
1405
- log.error(format_log("E037") + f" (ref={l_value}, action='{action}')")
1406
- result["error_code"] = "E037"
1407
- return result
1443
+ return self._recover_web_ref(handler, action, l_value, extra_value, result)
1408
1444
 
1409
1445
  # Visual fallback can only CLICK (a VLM returns a point, nothing
1410
1446
  # else) — never engage it for input/hover/assert/etc.
1411
1447
  if element is None and self.platform == "web" and action == "click":
1412
- try:
1413
- from common.visual_fallback import visual_locate
1414
- screenshot_bytes = self.d.screenshot()
1415
- coords = visual_locate(
1416
- screenshot_bytes,
1417
- f"{l_type}={l_value}",
1418
- )
1419
- if coords:
1420
- cx, cy = coords
1421
- log.info(f"👁️ [Visual Fallback] Using VLM coordinates: ({cx}, {cy})")
1422
- # Click live so the session advances, but the visual
1423
- # fallback has NO DOM node — only a VLM screenshot hit —
1424
- # so there is no durable locator to emit. Persist a skip
1425
- # (with the coordinate kept as a comment hint) rather than
1426
- # a coordinate that will rot and pass green against the
1427
- # wrong pixel later.
1428
- self.d.mouse.click(cx, cy)
1429
- code_lines = [
1430
- f" with allure.step('Click: [{l_value}] (UNREPLAYABLE: visual)'):\n",
1431
- " import pytest\n",
1432
- f" # Located only via VLM screenshot at ({cx}, {cy}); no DOM locator exists.\n",
1433
- f" # d.mouse.click({cx}, {cy}) # original visual hit, reference only\n",
1434
- f" pytest.skip('Visual-fallback step for [{l_value}] has no durable locator; add a selector to make it replayable')\n",
1435
- ]
1436
- result["success"] = True
1437
- result["code_lines"] = code_lines
1438
- result["action_description"] = f"⏭️ [Visual Fallback] {l_value} unreplayable — skip emitted"
1439
- return result
1440
- except Exception as e:
1441
- log.warning(f"⚠️ [Visual Fallback] Attempt failed: {e}")
1448
+ recovered = self._try_visual_fallback(l_type, l_value, result)
1449
+ if recovered is not None:
1450
+ return recovered
1442
1451
 
1443
1452
  if element is None:
1444
1453
  log.error(format_log("E033"))
@@ -43,15 +43,14 @@ AUTO_HEAL_MIN_CONFIDENCE = float(os.getenv("AUTO_HEAL_MIN_CONFIDENCE", 0.7))
43
43
  # ==========================================
44
44
  # 5. 自动化测试框架配置
45
45
  # ==========================================
46
- # 使用绝对路径,彻底杜绝不同命令路径下生成文件夹位置错乱的问题
47
- OUTPUT_SCRIPT_FILE = str(BASE_DIR / "test_cases" / "test_auto_generated.py")
48
-
49
46
  # 全局隐式等待时间,强转为 float 确保安全
50
47
  DEFAULT_TIMEOUT = float(os.getenv("DEFAULT_TIMEOUT", 30.0))
51
48
 
52
49
  # ==========================================
53
50
  # 6. App 环境配置
54
51
  # ==========================================
52
+ # 用户填入各环境/平台的 App 包名或 URL 后,launch_app 会自动拉起;默认留空=不自动启动。
53
+ # 这是文档化的用户旋钮(见 docs/capability-matrix.md),不是死代码。
55
54
  APP_ENV_CONFIG = {
56
55
  "dev": {
57
56
  "android": "",
@@ -68,7 +67,6 @@ CACHE_ENABLED = str(os.getenv("CACHE_ENABLED", "True")).lower() in ('true', '1',
68
67
  CACHE_DIR = str(BASE_DIR / '.cache')
69
68
  CACHE_TTL_DAYS = int(os.getenv("CACHE_TTL_DAYS", 7))
70
69
  CACHE_MAX_SIZE_MB = int(os.getenv("CACHE_MAX_SIZE_MB", 100))
71
- CACHE_COMPRESSION = str(os.getenv("CACHE_COMPRESSION", "False")).lower() in ('true', '1', 't', 'yes')
72
70
 
73
71
  CACHE_SIMILARITY_THRESHOLD = float(os.getenv("CACHE_SIMILARITY_THRESHOLD", "0.90"))
74
72
  CACHE_EXACT_MATCH_THRESHOLD = float(os.getenv("CACHE_EXACT_MATCH_THRESHOLD", "0.98"))
@@ -0,0 +1,23 @@
1
+ from pathlib import Path
2
+
3
+
4
+ def resolve_dotenv_path(project_root: Path):
5
+ project_root = Path(project_root).resolve()
6
+ for candidate_root in [project_root, *project_root.parents]:
7
+ try:
8
+ entries = {path.name.lower(): path for path in candidate_root.iterdir()}
9
+ except OSError:
10
+ continue
11
+ for file_name in (".env", ".ENV"):
12
+ candidate = entries.get(file_name.lower())
13
+ if candidate and candidate.is_file():
14
+ return candidate
15
+ return project_root / ".env"
16
+
17
+
18
+ def safe_load_dotenv(dotenv_path: Path, override: bool = False) -> bool:
19
+ # python-dotenv is a hard core dependency (pyproject.toml), so import it
20
+ # directly — the old hand-rolled fallback parser could never run.
21
+ from dotenv import load_dotenv
22
+
23
+ return bool(load_dotenv(dotenv_path=dotenv_path, override=override))
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "screenforge"
3
- version = "0.6.1"
3
+ version = "0.7.0"
4
4
  description = "AI-driven cross-platform UI automation engine with test script generation"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -71,6 +71,15 @@ ignore_missing_imports = true
71
71
  no_implicit_optional = false
72
72
  exclude = ["test_cases/", "scripts/", "\\.worktrees/"]
73
73
 
74
+ # numpy 2.4+ ships PEP 695 `type X = ...` in its bundled stubs, which mypy rejects
75
+ # under our 3.11 target (requires-python >=3.11, CI matrix tests 3.11). We don't
76
+ # type-check against numpy (cache_manager.py array math only) — skip its stubs so a
77
+ # third-party stub's syntax can't break the whole run. Treats numpy as Any.
78
+ [[tool.mypy.overrides]]
79
+ module = ["numpy.*"]
80
+ follow_imports = "skip"
81
+ follow_imports_for_stubs = true
82
+
74
83
  [[tool.mypy.overrides]]
75
84
  module = [
76
85
  "common.runtime_modes",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: screenforge
3
- Version: 0.6.1
3
+ Version: 0.7.0
4
4
  Summary: AI-driven cross-platform UI automation engine with test script generation
5
5
  License: MIT
6
6
  Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
@@ -119,6 +119,7 @@ Each step: **inspect → decide → act → verify**. The AI decides, ScreenForg
119
119
  - **MCP server**: Any MCP-compatible Agent can drive ScreenForge natively
120
120
  - **Structured output**: JSON Lines events + `report/runs/<id>/` artifacts for CI integration
121
121
  - **Live Mirror playground**: Watch the generated pytest code grow line-by-line beside a live screenshot as the test runs — `screenforge --playground`. See the [Playground Guide](https://github.com/jhinzzz/ScreenForge/blob/main/docs/playground-guide.md)
122
+ - **Execution-replay review report**: Run any test with `REVIEW_RECORD=1` and get a self-contained, shareable offline `report.html` — a drag-timeline of each step's screenshot + the test source line that triggered it + the DOM tree, plus a `video.gif` filmstrip. Default off, zero overhead. (Web verified; mobile registry/recording seam in place but unverified.)
122
123
 
123
124
  ## Agent Integration (Claude Code / Cursor / Codex)
124
125
 
@@ -66,6 +66,7 @@ tests/test_ai_autonomous.py
66
66
  tests/test_ai_brain.py
67
67
  tests/test_ai_heal.py
68
68
  tests/test_android_smoke_live.py
69
+ tests/test_cache_hash.py
69
70
  tests/test_cache_manager.py
70
71
  tests/test_capabilities.py
71
72
  tests/test_cli_action_json.py
@@ -87,6 +88,16 @@ tests/test_parser.py
87
88
  tests/test_playground_app.py
88
89
  tests/test_playground_sink.py
89
90
  tests/test_playground_sink_integration.py
91
+ tests/test_review_cases.py
92
+ tests/test_review_conftest_gating.py
93
+ tests/test_review_filmstrip.py
94
+ tests/test_review_html.py
95
+ tests/test_review_json.py
96
+ tests/test_review_locate_frame.py
97
+ tests/test_review_patching.py
98
+ tests/test_review_recorder.py
99
+ tests/test_review_screenshot.py
100
+ tests/test_review_smoke_live.py
90
101
  tests/test_run_reporter.py
91
102
  tests/test_run_resume.py
92
103
  tests/test_runtime_modes.py