frontend-perception-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- frontend_perception_engine-0.1.0.dist-info/METADATA +235 -0
- frontend_perception_engine-0.1.0.dist-info/RECORD +50 -0
- frontend_perception_engine-0.1.0.dist-info/WHEEL +5 -0
- frontend_perception_engine-0.1.0.dist-info/entry_points.txt +2 -0
- frontend_perception_engine-0.1.0.dist-info/top_level.txt +1 -0
- navigation/__init__.py +1 -0
- navigation/browser_use/__init__.py +17 -0
- navigation/browser_use/agent_runner.py +165 -0
- navigation/browser_use/hints.py +155 -0
- navigation/browser_use/integration.py +46 -0
- navigation/browser_use/llm.py +70 -0
- navigation/codeGraph/__init__.py +4 -0
- navigation/codeGraph/crg_impl.py +170 -0
- navigation/codeGraph/factory.py +24 -0
- navigation/codeGraph/interface.py +78 -0
- navigation/codeGraph/null_impl.py +47 -0
- navigation/mcp/__init__.py +4 -0
- navigation/mcp/__main__.py +4 -0
- navigation/mcp/diff.py +28 -0
- navigation/mcp/envelope.py +50 -0
- navigation/mcp/handlers.py +791 -0
- navigation/mcp/instructions.py +36 -0
- navigation/mcp/resources.py +82 -0
- navigation/mcp/scan_registry.py +47 -0
- navigation/mcp/server.py +229 -0
- navigation/mcp/session_store.py +87 -0
- navigation/mcp/tools.py +332 -0
- navigation/perception/__init__.py +76 -0
- navigation/perception/artifacts.py +19 -0
- navigation/perception/auth_gate.py +57 -0
- navigation/perception/budget.py +55 -0
- navigation/perception/cdp_hub.py +122 -0
- navigation/perception/dev_insights.py +625 -0
- navigation/perception/exploration.py +99 -0
- navigation/perception/feature_flags.py +83 -0
- navigation/perception/file_upload.py +84 -0
- navigation/perception/flow_graph.py +121 -0
- navigation/perception/form_probe.py +148 -0
- navigation/perception/iframe_context.py +76 -0
- navigation/perception/observation.py +97 -0
- navigation/perception/preflight.py +91 -0
- navigation/perception/rich_editors.py +90 -0
- navigation/perception/route_guards.py +136 -0
- navigation/perception/runner.py +138 -0
- navigation/perception/scan.py +74 -0
- navigation/perception/scripted_actions.py +67 -0
- navigation/perception/state_manager.py +114 -0
- navigation/perception/verification.py +117 -0
- navigation/perception/virtual_scroll.py +84 -0
- navigation/perception/websocket_observer.py +79 -0
|
@@ -0,0 +1,791 @@
|
|
|
1
|
+
"""MCP tool handlers — thin wrappers over navigation.perception engine."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import urllib.error
|
|
5
|
+
import urllib.request
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import urljoin
|
|
8
|
+
|
|
9
|
+
from navigation.perception.budget import OutputBudget, apply_observation_budget
|
|
10
|
+
from navigation.perception.dev_insights import collect_dev_insights_during
|
|
11
|
+
from navigation.perception.preflight import preflight_check, wait_for_page_ready
|
|
12
|
+
from navigation.perception.scan import scan_page
|
|
13
|
+
from navigation.perception.verification import SuccessCriteria, evaluate_js, verify
|
|
14
|
+
|
|
15
|
+
from .diff import diff_observations
|
|
16
|
+
from .envelope import agent_summary_from_observation, make_envelope
|
|
17
|
+
from .scan_registry import ScanRegistry
|
|
18
|
+
from .session_store import SessionStore
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _resolve_url(base: str, url: str) -> str:
|
|
22
|
+
if url.startswith("http://") or url.startswith("https://"):
|
|
23
|
+
return url
|
|
24
|
+
return urljoin(base.rstrip("/") + "/", url.lstrip("/"))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _criteria_from_dict(criteria_raw: dict[str, Any]) -> SuccessCriteria:
|
|
28
|
+
return SuccessCriteria(
|
|
29
|
+
url_contains=list(criteria_raw.get("url_contains") or []),
|
|
30
|
+
url_not_contains=list(criteria_raw.get("url_not_contains") or []),
|
|
31
|
+
url_regex=criteria_raw.get("url_regex"),
|
|
32
|
+
text_contains=list(criteria_raw.get("text_contains") or []),
|
|
33
|
+
text_absent=list(criteria_raw.get("text_absent") or []),
|
|
34
|
+
js_assertions=list(criteria_raw.get("js_assertions") or []),
|
|
35
|
+
accept_urls=list(criteria_raw.get("accept_urls") or []),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _budget_from_args(budget_raw: dict[str, Any] | None) -> OutputBudget:
|
|
40
|
+
budget_raw = budget_raw or {}
|
|
41
|
+
return OutputBudget(
|
|
42
|
+
max_a11y_chars=int(budget_raw.get("max_a11y_chars", 4000)),
|
|
43
|
+
max_dom_chars=int(budget_raw.get("max_dom_chars", 4000)),
|
|
44
|
+
max_list_items=int(budget_raw.get("max_list_items", 30)),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _detail_mode(arguments: dict[str, Any]) -> str:
|
|
49
|
+
detail = str(arguments.get("detail") or "full").strip().lower()
|
|
50
|
+
return detail if detail in {"full", "summary_only"} else "full"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _observation_payload(obs_dict: dict[str, Any], summary: dict[str, Any], detail: str) -> dict[str, Any]:
|
|
54
|
+
data: dict[str, Any] = {"agent_summary": summary}
|
|
55
|
+
if detail == "full":
|
|
56
|
+
data["observation"] = obs_dict
|
|
57
|
+
return data
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _require_session(store: SessionStore, session_id: str) -> tuple[Any | None, dict[str, Any] | None]:
|
|
61
|
+
if not session_id:
|
|
62
|
+
return None, make_envelope("", ok=False, error="session_id required")
|
|
63
|
+
try:
|
|
64
|
+
return store.require(session_id), None
|
|
65
|
+
except KeyError as exc:
|
|
66
|
+
return None, make_envelope("", ok=False, error=str(exc))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def handle_health(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
70
|
+
url = str(arguments.get("url") or "http://localhost:5173")
|
|
71
|
+
reachable = False
|
|
72
|
+
status: int | None = None
|
|
73
|
+
error: str | None = None
|
|
74
|
+
try:
|
|
75
|
+
with urllib.request.urlopen(url, timeout=3) as resp:
|
|
76
|
+
status = resp.status
|
|
77
|
+
reachable = status == 200
|
|
78
|
+
except urllib.error.HTTPError as exc:
|
|
79
|
+
status = exc.code
|
|
80
|
+
reachable = exc.code < 500
|
|
81
|
+
error = str(exc)
|
|
82
|
+
except Exception as exc:
|
|
83
|
+
error = str(exc)
|
|
84
|
+
|
|
85
|
+
return make_envelope(
|
|
86
|
+
"perception_health",
|
|
87
|
+
ok=reachable,
|
|
88
|
+
url=url,
|
|
89
|
+
error=None if reachable else (error or f"unreachable (status={status})"),
|
|
90
|
+
data={"reachable": reachable, "status": status},
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
async def handle_session_start(
|
|
95
|
+
store: SessionStore,
|
|
96
|
+
arguments: dict[str, Any],
|
|
97
|
+
) -> dict[str, Any]:
|
|
98
|
+
base_url = str(arguments.get("base_url") or "http://localhost:5173")
|
|
99
|
+
headless = bool(arguments.get("headless", True))
|
|
100
|
+
viewport = arguments.get("viewport") or {}
|
|
101
|
+
try:
|
|
102
|
+
rec = await store.start(
|
|
103
|
+
base_url=base_url,
|
|
104
|
+
headless=headless,
|
|
105
|
+
viewport_width=int(viewport.get("width", 1920)),
|
|
106
|
+
viewport_height=int(viewport.get("height", 1080)),
|
|
107
|
+
)
|
|
108
|
+
except Exception as exc:
|
|
109
|
+
return make_envelope("perception_session_start", ok=False, error=str(exc))
|
|
110
|
+
|
|
111
|
+
return make_envelope(
|
|
112
|
+
"perception_session_start",
|
|
113
|
+
session_id=rec.session_id,
|
|
114
|
+
run_id=rec.current_run_id,
|
|
115
|
+
url=rec.base_url,
|
|
116
|
+
data={
|
|
117
|
+
"session_id": rec.session_id,
|
|
118
|
+
"run_id": rec.current_run_id,
|
|
119
|
+
"base_url": rec.base_url,
|
|
120
|
+
"artifacts_dir": str(rec.artifacts_dir),
|
|
121
|
+
},
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
async def handle_session_end(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
126
|
+
session_id = str(arguments.get("session_id") or "")
|
|
127
|
+
if not session_id:
|
|
128
|
+
return make_envelope("perception_session_end", ok=False, error="session_id required")
|
|
129
|
+
ended = await store.end(session_id)
|
|
130
|
+
return make_envelope(
|
|
131
|
+
"perception_session_end",
|
|
132
|
+
ok=ended,
|
|
133
|
+
session_id=session_id,
|
|
134
|
+
error=None if ended else f"unknown session_id: {session_id}",
|
|
135
|
+
data={"ended": ended},
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def handle_navigate_and_observe(
|
|
140
|
+
store: SessionStore,
|
|
141
|
+
scans: ScanRegistry,
|
|
142
|
+
arguments: dict[str, Any],
|
|
143
|
+
) -> dict[str, Any]:
|
|
144
|
+
session_id = str(arguments.get("session_id") or "")
|
|
145
|
+
url_arg = str(arguments.get("url") or "")
|
|
146
|
+
if not session_id or not url_arg:
|
|
147
|
+
return make_envelope(
|
|
148
|
+
"perception_navigate_and_observe",
|
|
149
|
+
ok=False,
|
|
150
|
+
error="session_id and url required",
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
rec = store.require(session_id)
|
|
155
|
+
except KeyError as exc:
|
|
156
|
+
return make_envelope("perception_navigate_and_observe", ok=False, error=str(exc))
|
|
157
|
+
|
|
158
|
+
target = _resolve_url(rec.base_url, url_arg)
|
|
159
|
+
include_screenshot = bool(arguments.get("include_screenshot", True))
|
|
160
|
+
detail = _detail_mode(arguments)
|
|
161
|
+
budget_raw = arguments.get("budget") or {}
|
|
162
|
+
budget = OutputBudget(
|
|
163
|
+
max_a11y_chars=int(budget_raw.get("max_a11y_chars", 4000)),
|
|
164
|
+
max_dom_chars=int(budget_raw.get("max_dom_chars", 4000)),
|
|
165
|
+
max_list_items=int(budget_raw.get("max_list_items", 30)),
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
images_dir = rec.artifacts_dir / "images" if include_screenshot else None
|
|
169
|
+
result = await scan_page(
|
|
170
|
+
rec.browser,
|
|
171
|
+
target,
|
|
172
|
+
images_dir=images_dir,
|
|
173
|
+
name=f"scan-{rec.run_counter}",
|
|
174
|
+
budget=budget,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if not result.ok:
|
|
178
|
+
return make_envelope(
|
|
179
|
+
"perception_navigate_and_observe",
|
|
180
|
+
ok=False,
|
|
181
|
+
session_id=session_id,
|
|
182
|
+
run_id=rec.current_run_id,
|
|
183
|
+
url=target,
|
|
184
|
+
error=result.error,
|
|
185
|
+
degraded=result.degraded,
|
|
186
|
+
data={"preflight": result.preflight.to_dict() if result.preflight else None},
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
obs_dict = result.observation.to_dict() if result.observation else {}
|
|
190
|
+
scan_rec = scans.register(
|
|
191
|
+
session_id=session_id,
|
|
192
|
+
run_id=rec.current_run_id,
|
|
193
|
+
url=result.url,
|
|
194
|
+
observation=obs_dict,
|
|
195
|
+
)
|
|
196
|
+
summary = agent_summary_from_observation(obs_dict)
|
|
197
|
+
|
|
198
|
+
return make_envelope(
|
|
199
|
+
"perception_navigate_and_observe",
|
|
200
|
+
session_id=session_id,
|
|
201
|
+
run_id=rec.current_run_id,
|
|
202
|
+
scan_id=scan_rec.scan_id,
|
|
203
|
+
url=result.url,
|
|
204
|
+
degraded=result.degraded,
|
|
205
|
+
data={
|
|
206
|
+
"scan_id": scan_rec.scan_id,
|
|
207
|
+
**_observation_payload(obs_dict, summary, detail),
|
|
208
|
+
"detail": detail,
|
|
209
|
+
"preflight": result.preflight.to_dict() if result.preflight else None,
|
|
210
|
+
},
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
async def handle_verify(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
215
|
+
session_id = str(arguments.get("session_id") or "")
|
|
216
|
+
criteria_raw = arguments.get("criteria") or {}
|
|
217
|
+
if not session_id:
|
|
218
|
+
return make_envelope("perception_verify", ok=False, error="session_id required")
|
|
219
|
+
|
|
220
|
+
try:
|
|
221
|
+
rec = store.require(session_id)
|
|
222
|
+
except KeyError as exc:
|
|
223
|
+
return make_envelope("perception_verify", ok=False, error=str(exc))
|
|
224
|
+
|
|
225
|
+
criteria = _criteria_from_dict(criteria_raw)
|
|
226
|
+
vr = await verify(rec.browser, criteria)
|
|
227
|
+
return make_envelope(
|
|
228
|
+
"perception_verify",
|
|
229
|
+
session_id=session_id,
|
|
230
|
+
run_id=rec.current_run_id,
|
|
231
|
+
url=vr.url,
|
|
232
|
+
ok=True,
|
|
233
|
+
data={
|
|
234
|
+
"verified": vr.ok,
|
|
235
|
+
"url": vr.url,
|
|
236
|
+
"reasons": vr.reasons,
|
|
237
|
+
"auto_merged": vr.auto_merged,
|
|
238
|
+
"feedback": vr.feedback(),
|
|
239
|
+
},
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
async def handle_execute_script(
|
|
244
|
+
store: SessionStore,
|
|
245
|
+
scans: ScanRegistry,
|
|
246
|
+
arguments: dict[str, Any],
|
|
247
|
+
) -> dict[str, Any]:
|
|
248
|
+
session_id = str(arguments.get("session_id") or "")
|
|
249
|
+
script = str(arguments.get("script") or "")
|
|
250
|
+
if not session_id or not script:
|
|
251
|
+
return make_envelope(
|
|
252
|
+
"perception_execute_script",
|
|
253
|
+
ok=False,
|
|
254
|
+
error="session_id and script required",
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
try:
|
|
258
|
+
rec = store.require(session_id)
|
|
259
|
+
except KeyError as exc:
|
|
260
|
+
return make_envelope("perception_execute_script", ok=False, error=str(exc))
|
|
261
|
+
|
|
262
|
+
capture = bool(arguments.get("capture_insights_during", True))
|
|
263
|
+
scan_before = scans.register(
|
|
264
|
+
session_id=session_id,
|
|
265
|
+
run_id=rec.current_run_id,
|
|
266
|
+
url=await _current_url(rec.browser),
|
|
267
|
+
observation={"phase": "before_execute"},
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
script_ok = False
|
|
271
|
+
return_value: Any = None
|
|
272
|
+
insights_dict: dict[str, Any] | None = None
|
|
273
|
+
error: str | None = None
|
|
274
|
+
|
|
275
|
+
async def _run() -> None:
|
|
276
|
+
nonlocal return_value
|
|
277
|
+
return_value = await evaluate_js(rec.browser, script)
|
|
278
|
+
|
|
279
|
+
try:
|
|
280
|
+
if capture:
|
|
281
|
+
insights = await collect_dev_insights_during(rec.browser, _run)
|
|
282
|
+
insights_dict = insights.to_dict()
|
|
283
|
+
else:
|
|
284
|
+
await _run()
|
|
285
|
+
script_ok = True
|
|
286
|
+
except Exception as exc:
|
|
287
|
+
error = str(exc)
|
|
288
|
+
|
|
289
|
+
await wait_for_page_ready(rec.browser, timeout=5.0)
|
|
290
|
+
from navigation.perception.observation import collect_observation
|
|
291
|
+
|
|
292
|
+
obs = await collect_observation(
|
|
293
|
+
rec.browser,
|
|
294
|
+
images_dir=rec.artifacts_dir / "images",
|
|
295
|
+
name=f"after-exec-{rec.run_counter}",
|
|
296
|
+
)
|
|
297
|
+
scan_after = scans.register(
|
|
298
|
+
session_id=session_id,
|
|
299
|
+
run_id=rec.current_run_id,
|
|
300
|
+
url=obs.url,
|
|
301
|
+
observation=obs.to_dict(),
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
return make_envelope(
|
|
305
|
+
"perception_execute_script",
|
|
306
|
+
ok=script_ok,
|
|
307
|
+
session_id=session_id,
|
|
308
|
+
run_id=rec.current_run_id,
|
|
309
|
+
scan_id=scan_after.scan_id,
|
|
310
|
+
url=obs.url,
|
|
311
|
+
error=error,
|
|
312
|
+
degraded=obs.degraded,
|
|
313
|
+
data={
|
|
314
|
+
"script_ok": script_ok,
|
|
315
|
+
"return_value": return_value,
|
|
316
|
+
"scan_id_before": scan_before.scan_id,
|
|
317
|
+
"scan_id_after": scan_after.scan_id,
|
|
318
|
+
"dev_insights": insights_dict,
|
|
319
|
+
"agent_summary": agent_summary_from_observation(obs.to_dict()),
|
|
320
|
+
},
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
async def _current_url(browser: Any) -> str:
|
|
325
|
+
from navigation.perception.verification import read_current_url
|
|
326
|
+
|
|
327
|
+
return await read_current_url(browser)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
async def _observe_and_register(
|
|
331
|
+
rec: Any,
|
|
332
|
+
scans: ScanRegistry,
|
|
333
|
+
session_id: str,
|
|
334
|
+
*,
|
|
335
|
+
include_screenshot: bool = True,
|
|
336
|
+
budget: OutputBudget | None = None,
|
|
337
|
+
name: str | None = None,
|
|
338
|
+
) -> tuple[dict[str, Any], Any]:
|
|
339
|
+
from navigation.perception.observation import collect_observation
|
|
340
|
+
|
|
341
|
+
images_dir = rec.artifacts_dir / "images" if include_screenshot else None
|
|
342
|
+
obs = await collect_observation(
|
|
343
|
+
rec.browser,
|
|
344
|
+
images_dir=images_dir,
|
|
345
|
+
name=name or f"observe-{rec.run_counter}",
|
|
346
|
+
)
|
|
347
|
+
obs_dict = obs.to_dict()
|
|
348
|
+
if budget:
|
|
349
|
+
obs_dict = apply_observation_budget(obs_dict, budget)
|
|
350
|
+
scan_rec = scans.register(
|
|
351
|
+
session_id=session_id,
|
|
352
|
+
run_id=rec.current_run_id,
|
|
353
|
+
url=obs.url,
|
|
354
|
+
observation=obs_dict,
|
|
355
|
+
)
|
|
356
|
+
return obs_dict, scan_rec
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
async def handle_navigate(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
360
|
+
session_id = str(arguments.get("session_id") or "")
|
|
361
|
+
url_arg = str(arguments.get("url") or "")
|
|
362
|
+
if not session_id or not url_arg:
|
|
363
|
+
return make_envelope("perception_navigate", ok=False, error="session_id and url required")
|
|
364
|
+
|
|
365
|
+
rec, err = _require_session(store, session_id)
|
|
366
|
+
if err:
|
|
367
|
+
return {**err, "tool": "perception_navigate"}
|
|
368
|
+
|
|
369
|
+
target = _resolve_url(rec.base_url, url_arg)
|
|
370
|
+
pf = await preflight_check(rec.browser, target)
|
|
371
|
+
return make_envelope(
|
|
372
|
+
"perception_navigate",
|
|
373
|
+
ok=pf.ok,
|
|
374
|
+
session_id=session_id,
|
|
375
|
+
run_id=rec.current_run_id,
|
|
376
|
+
url=pf.url or target,
|
|
377
|
+
error=pf.error,
|
|
378
|
+
degraded=pf.degraded,
|
|
379
|
+
data={"preflight": pf.to_dict()},
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
async def handle_observe(
|
|
384
|
+
store: SessionStore,
|
|
385
|
+
scans: ScanRegistry,
|
|
386
|
+
arguments: dict[str, Any],
|
|
387
|
+
) -> dict[str, Any]:
|
|
388
|
+
session_id = str(arguments.get("session_id") or "")
|
|
389
|
+
if not session_id:
|
|
390
|
+
return make_envelope("perception_observe", ok=False, error="session_id required")
|
|
391
|
+
|
|
392
|
+
rec, err = _require_session(store, session_id)
|
|
393
|
+
if err:
|
|
394
|
+
return {**err, "tool": "perception_observe"}
|
|
395
|
+
|
|
396
|
+
include_screenshot = bool(arguments.get("include_screenshot", True))
|
|
397
|
+
detail = _detail_mode(arguments)
|
|
398
|
+
budget = _budget_from_args(arguments.get("budget"))
|
|
399
|
+
obs_dict, scan_rec = await _observe_and_register(
|
|
400
|
+
rec,
|
|
401
|
+
scans,
|
|
402
|
+
session_id,
|
|
403
|
+
include_screenshot=include_screenshot,
|
|
404
|
+
budget=budget,
|
|
405
|
+
)
|
|
406
|
+
summary = agent_summary_from_observation(obs_dict)
|
|
407
|
+
return make_envelope(
|
|
408
|
+
"perception_observe",
|
|
409
|
+
session_id=session_id,
|
|
410
|
+
run_id=rec.current_run_id,
|
|
411
|
+
scan_id=scan_rec.scan_id,
|
|
412
|
+
url=obs_dict.get("url") or "",
|
|
413
|
+
degraded=list(obs_dict.get("degraded") or []),
|
|
414
|
+
data={
|
|
415
|
+
"scan_id": scan_rec.scan_id,
|
|
416
|
+
**_observation_payload(obs_dict, summary, detail),
|
|
417
|
+
"detail": detail,
|
|
418
|
+
},
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
async def handle_execute_actions(
|
|
423
|
+
store: SessionStore,
|
|
424
|
+
scans: ScanRegistry,
|
|
425
|
+
arguments: dict[str, Any],
|
|
426
|
+
) -> dict[str, Any]:
|
|
427
|
+
from navigation.perception.scripted_actions import (
|
|
428
|
+
click_button_text,
|
|
429
|
+
click_link_text,
|
|
430
|
+
set_input_by_label,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
session_id = str(arguments.get("session_id") or "")
|
|
434
|
+
actions = arguments.get("actions") or []
|
|
435
|
+
if not session_id:
|
|
436
|
+
return make_envelope("perception_execute_actions", ok=False, error="session_id required")
|
|
437
|
+
if not isinstance(actions, list) or not actions:
|
|
438
|
+
return make_envelope("perception_execute_actions", ok=False, error="actions list required")
|
|
439
|
+
|
|
440
|
+
rec, err = _require_session(store, session_id)
|
|
441
|
+
if err:
|
|
442
|
+
return {**err, "tool": "perception_execute_actions"}
|
|
443
|
+
|
|
444
|
+
capture = bool(arguments.get("capture_insights_during", True))
|
|
445
|
+
results: list[dict[str, Any]] = []
|
|
446
|
+
insights_dict: dict[str, Any] | None = None
|
|
447
|
+
all_ok = True
|
|
448
|
+
error: str | None = None
|
|
449
|
+
|
|
450
|
+
async def _run_actions() -> None:
|
|
451
|
+
nonlocal all_ok, error
|
|
452
|
+
for idx, action in enumerate(actions):
|
|
453
|
+
if not isinstance(action, dict):
|
|
454
|
+
all_ok = False
|
|
455
|
+
results.append({"index": idx, "ok": False, "error": "action must be object"})
|
|
456
|
+
error = "invalid action"
|
|
457
|
+
break
|
|
458
|
+
kind = str(action.get("type") or "")
|
|
459
|
+
ok = False
|
|
460
|
+
if kind == "click_button":
|
|
461
|
+
ok = await click_button_text(rec.browser, str(action.get("text") or ""))
|
|
462
|
+
elif kind == "click_link":
|
|
463
|
+
ok = await click_link_text(rec.browser, str(action.get("text") or ""))
|
|
464
|
+
elif kind == "set_input":
|
|
465
|
+
ok = await set_input_by_label(
|
|
466
|
+
rec.browser,
|
|
467
|
+
str(action.get("label") or ""),
|
|
468
|
+
str(action.get("value") or ""),
|
|
469
|
+
)
|
|
470
|
+
else:
|
|
471
|
+
results.append({"index": idx, "type": kind, "ok": False, "error": "unknown action type"})
|
|
472
|
+
all_ok = False
|
|
473
|
+
error = f"unknown action type: {kind}"
|
|
474
|
+
break
|
|
475
|
+
results.append({"index": idx, "type": kind, "ok": ok})
|
|
476
|
+
if not ok:
|
|
477
|
+
all_ok = False
|
|
478
|
+
error = f"action {idx} ({kind}) failed"
|
|
479
|
+
break
|
|
480
|
+
await wait_for_page_ready(rec.browser, timeout=3.0)
|
|
481
|
+
|
|
482
|
+
try:
|
|
483
|
+
if capture:
|
|
484
|
+
insights = await collect_dev_insights_during(rec.browser, _run_actions)
|
|
485
|
+
insights_dict = insights.to_dict()
|
|
486
|
+
else:
|
|
487
|
+
await _run_actions()
|
|
488
|
+
except Exception as exc:
|
|
489
|
+
all_ok = False
|
|
490
|
+
error = str(exc)
|
|
491
|
+
|
|
492
|
+
obs_dict, scan_rec = await _observe_and_register(
|
|
493
|
+
rec,
|
|
494
|
+
scans,
|
|
495
|
+
session_id,
|
|
496
|
+
name=f"after-actions-{rec.run_counter}",
|
|
497
|
+
)
|
|
498
|
+
return make_envelope(
|
|
499
|
+
"perception_execute_actions",
|
|
500
|
+
ok=all_ok,
|
|
501
|
+
session_id=session_id,
|
|
502
|
+
run_id=rec.current_run_id,
|
|
503
|
+
scan_id=scan_rec.scan_id,
|
|
504
|
+
url=obs_dict.get("url") or "",
|
|
505
|
+
error=error,
|
|
506
|
+
degraded=list(obs_dict.get("degraded") or []),
|
|
507
|
+
data={
|
|
508
|
+
"actions_ok": all_ok,
|
|
509
|
+
"action_results": results,
|
|
510
|
+
"scan_id": scan_rec.scan_id,
|
|
511
|
+
"dev_insights": insights_dict,
|
|
512
|
+
"agent_summary": agent_summary_from_observation(obs_dict),
|
|
513
|
+
},
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
async def handle_diff(scans: ScanRegistry, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
518
|
+
before_id = str(arguments.get("scan_id_before") or "")
|
|
519
|
+
after_id = str(arguments.get("scan_id_after") or "")
|
|
520
|
+
if not before_id or not after_id:
|
|
521
|
+
return make_envelope(
|
|
522
|
+
"perception_diff",
|
|
523
|
+
ok=False,
|
|
524
|
+
error="scan_id_before and scan_id_after required",
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
before_rec = scans.get(before_id)
|
|
528
|
+
after_rec = scans.get(after_id)
|
|
529
|
+
if before_rec is None or after_rec is None:
|
|
530
|
+
missing = []
|
|
531
|
+
if before_rec is None:
|
|
532
|
+
missing.append(before_id)
|
|
533
|
+
if after_rec is None:
|
|
534
|
+
missing.append(after_id)
|
|
535
|
+
return make_envelope(
|
|
536
|
+
"perception_diff",
|
|
537
|
+
ok=False,
|
|
538
|
+
error=f"unknown scan_id(s): {', '.join(missing)}",
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
diff = diff_observations(before_rec.observation, after_rec.observation)
|
|
542
|
+
return make_envelope(
|
|
543
|
+
"perception_diff",
|
|
544
|
+
ok=True,
|
|
545
|
+
session_id=after_rec.session_id,
|
|
546
|
+
data={
|
|
547
|
+
"scan_id_before": before_id,
|
|
548
|
+
"scan_id_after": after_id,
|
|
549
|
+
"diff": diff,
|
|
550
|
+
"has_changes": diff["url_changed"] or diff["dom_text_changed"] or bool(diff["new_blocking_issues"]),
|
|
551
|
+
},
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
async def handle_auth_gate(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
556
|
+
from navigation.perception.auth_gate import check_auth_gate
|
|
557
|
+
|
|
558
|
+
session_id = str(arguments.get("session_id") or "")
|
|
559
|
+
if not session_id:
|
|
560
|
+
return make_envelope("perception_auth_gate", ok=False, error="session_id required")
|
|
561
|
+
|
|
562
|
+
rec, err = _require_session(store, session_id)
|
|
563
|
+
if err:
|
|
564
|
+
return {**err, "tool": "perception_auth_gate"}
|
|
565
|
+
|
|
566
|
+
gate = await check_auth_gate(rec.browser)
|
|
567
|
+
return make_envelope(
|
|
568
|
+
"perception_auth_gate",
|
|
569
|
+
ok=True,
|
|
570
|
+
session_id=session_id,
|
|
571
|
+
run_id=rec.current_run_id,
|
|
572
|
+
url=gate.url,
|
|
573
|
+
data={
|
|
574
|
+
"requires_human": gate.requires_human,
|
|
575
|
+
"reason": gate.reason,
|
|
576
|
+
"gate": gate.to_dict(),
|
|
577
|
+
},
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
async def handle_probe_form(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
582
|
+
from navigation.perception.form_probe import probe_validation_form
|
|
583
|
+
|
|
584
|
+
session_id = str(arguments.get("session_id") or "")
|
|
585
|
+
form = str(arguments.get("form") or "validation")
|
|
586
|
+
if not session_id:
|
|
587
|
+
return make_envelope("perception_probe_form", ok=False, error="session_id required")
|
|
588
|
+
|
|
589
|
+
rec, err = _require_session(store, session_id)
|
|
590
|
+
if err:
|
|
591
|
+
return {**err, "tool": "perception_probe_form"}
|
|
592
|
+
|
|
593
|
+
if form != "validation":
|
|
594
|
+
return make_envelope(
|
|
595
|
+
"perception_probe_form",
|
|
596
|
+
ok=False,
|
|
597
|
+
session_id=session_id,
|
|
598
|
+
error=f"unsupported form probe: {form}",
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
probe = await probe_validation_form(rec.browser, rec.base_url)
|
|
602
|
+
return make_envelope(
|
|
603
|
+
"perception_probe_form",
|
|
604
|
+
ok=probe.ok,
|
|
605
|
+
session_id=session_id,
|
|
606
|
+
run_id=rec.current_run_id,
|
|
607
|
+
url=probe.form_url,
|
|
608
|
+
error=probe.error,
|
|
609
|
+
data={"probe": probe.to_dict()},
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
async def handle_probe_guards(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
614
|
+
from navigation.perception.route_guards import probe_maze_guards, probe_route_guard
|
|
615
|
+
|
|
616
|
+
session_id = str(arguments.get("session_id") or "")
|
|
617
|
+
if not session_id:
|
|
618
|
+
return make_envelope("perception_probe_guards", ok=False, error="session_id required")
|
|
619
|
+
|
|
620
|
+
rec, err = _require_session(store, session_id)
|
|
621
|
+
if err:
|
|
622
|
+
return {**err, "tool": "perception_probe_guards"}
|
|
623
|
+
|
|
624
|
+
mode = str(arguments.get("mode") or "maze")
|
|
625
|
+
if mode == "maze":
|
|
626
|
+
result = await probe_maze_guards(rec.browser, rec.base_url)
|
|
627
|
+
elif mode == "routes":
|
|
628
|
+
routes = arguments.get("routes") or []
|
|
629
|
+
guards = []
|
|
630
|
+
for spec in routes:
|
|
631
|
+
if not isinstance(spec, dict):
|
|
632
|
+
continue
|
|
633
|
+
guard = await probe_route_guard(
|
|
634
|
+
rec.browser,
|
|
635
|
+
rec.base_url,
|
|
636
|
+
str(spec.get("route") or "/"),
|
|
637
|
+
expected_redirect=spec.get("expected_redirect"),
|
|
638
|
+
requires_auth=bool(spec.get("requires_auth", False)),
|
|
639
|
+
requires_role=spec.get("requires_role"),
|
|
640
|
+
)
|
|
641
|
+
guards.append(guard)
|
|
642
|
+
from navigation.perception.route_guards import GuardProbeResult
|
|
643
|
+
|
|
644
|
+
result = GuardProbeResult(ok=bool(guards), guards=guards)
|
|
645
|
+
else:
|
|
646
|
+
return make_envelope(
|
|
647
|
+
"perception_probe_guards",
|
|
648
|
+
ok=False,
|
|
649
|
+
session_id=session_id,
|
|
650
|
+
error=f"unsupported mode: {mode}",
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
return make_envelope(
|
|
654
|
+
"perception_probe_guards",
|
|
655
|
+
ok=result.ok,
|
|
656
|
+
session_id=session_id,
|
|
657
|
+
run_id=rec.current_run_id,
|
|
658
|
+
error=result.error,
|
|
659
|
+
data={"probe": result.to_dict()},
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
async def handle_state_save(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
664
|
+
session_id = str(arguments.get("session_id") or "")
|
|
665
|
+
state_id = str(arguments.get("state_id") or "")
|
|
666
|
+
if not session_id or not state_id:
|
|
667
|
+
return make_envelope(
|
|
668
|
+
"perception_state_save",
|
|
669
|
+
ok=False,
|
|
670
|
+
error="session_id and state_id required",
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
rec, err = _require_session(store, session_id)
|
|
674
|
+
if err:
|
|
675
|
+
return {**err, "tool": "perception_state_save"}
|
|
676
|
+
|
|
677
|
+
snap = await rec.state_manager.snapshot(rec.browser, state_id)
|
|
678
|
+
return make_envelope(
|
|
679
|
+
"perception_state_save",
|
|
680
|
+
ok=True,
|
|
681
|
+
session_id=session_id,
|
|
682
|
+
run_id=rec.current_run_id,
|
|
683
|
+
url=snap.url,
|
|
684
|
+
data={"state": snap.to_dict()},
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
async def handle_state_restore(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
689
|
+
session_id = str(arguments.get("session_id") or "")
|
|
690
|
+
state_id = str(arguments.get("state_id") or "")
|
|
691
|
+
if not session_id or not state_id:
|
|
692
|
+
return make_envelope(
|
|
693
|
+
"perception_state_restore",
|
|
694
|
+
ok=False,
|
|
695
|
+
error="session_id and state_id required",
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
rec, err = _require_session(store, session_id)
|
|
699
|
+
if err:
|
|
700
|
+
return {**err, "tool": "perception_state_restore"}
|
|
701
|
+
|
|
702
|
+
restored = await rec.state_manager.restore(rec.browser, state_id)
|
|
703
|
+
url = await _current_url(rec.browser) if restored else ""
|
|
704
|
+
return make_envelope(
|
|
705
|
+
"perception_state_restore",
|
|
706
|
+
ok=restored,
|
|
707
|
+
session_id=session_id,
|
|
708
|
+
run_id=rec.current_run_id,
|
|
709
|
+
url=url,
|
|
710
|
+
error=None if restored else f"unknown state_id: {state_id}",
|
|
711
|
+
data={"restored": restored, "state_id": state_id},
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
async def handle_state_list(store: SessionStore, arguments: dict[str, Any]) -> dict[str, Any]:
|
|
716
|
+
session_id = str(arguments.get("session_id") or "")
|
|
717
|
+
if not session_id:
|
|
718
|
+
return make_envelope("perception_state_list", ok=False, error="session_id required")
|
|
719
|
+
|
|
720
|
+
rec, err = _require_session(store, session_id)
|
|
721
|
+
if err:
|
|
722
|
+
return {**err, "tool": "perception_state_list"}
|
|
723
|
+
|
|
724
|
+
states = rec.state_manager.list_states()
|
|
725
|
+
return make_envelope(
|
|
726
|
+
"perception_state_list",
|
|
727
|
+
ok=True,
|
|
728
|
+
session_id=session_id,
|
|
729
|
+
run_id=rec.current_run_id,
|
|
730
|
+
data={"states": states, "count": len(states)},
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
async def handle_flow_describe(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
735
|
+
from navigation.perception.flow_graph import FLOWS
|
|
736
|
+
|
|
737
|
+
flow_name = arguments.get("flow_name")
|
|
738
|
+
if not flow_name:
|
|
739
|
+
flows = {name: FLOWS[name]().description for name in FLOWS}
|
|
740
|
+
return make_envelope(
|
|
741
|
+
"perception_flow_describe",
|
|
742
|
+
ok=True,
|
|
743
|
+
data={"flows": flows, "flow_names": list(FLOWS.keys())},
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
name = str(flow_name)
|
|
747
|
+
factory = FLOWS.get(name)
|
|
748
|
+
if factory is None:
|
|
749
|
+
return make_envelope(
|
|
750
|
+
"perception_flow_describe",
|
|
751
|
+
ok=False,
|
|
752
|
+
error=f"unknown flow: {name}",
|
|
753
|
+
data={"flow_names": list(FLOWS.keys())},
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
flow = factory()
|
|
757
|
+
return make_envelope(
|
|
758
|
+
"perception_flow_describe",
|
|
759
|
+
ok=True,
|
|
760
|
+
data={"flow": flow.to_dict()},
|
|
761
|
+
)
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
async def handle_code_context(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
765
|
+
from pathlib import Path
|
|
766
|
+
|
|
767
|
+
from navigation.codeGraph import create_code_graph
|
|
768
|
+
|
|
769
|
+
repo_root = Path(str(arguments.get("repo_root") or "")).resolve() if arguments.get("repo_root") else None
|
|
770
|
+
if repo_root is None:
|
|
771
|
+
# Default: sandbox sibling of src/
|
|
772
|
+
repo_root = Path(__file__).resolve().parents[3] / "sandbox"
|
|
773
|
+
|
|
774
|
+
enabled = bool(arguments.get("enabled", True))
|
|
775
|
+
query_type = str(arguments.get("query_type") or "stats")
|
|
776
|
+
query_kwargs = dict(arguments.get("query_kwargs") or {})
|
|
777
|
+
|
|
778
|
+
graph = create_code_graph(repo_root, enabled=enabled)
|
|
779
|
+
result = graph.query(query_type, **query_kwargs)
|
|
780
|
+
|
|
781
|
+
return make_envelope(
|
|
782
|
+
"perception_code_context",
|
|
783
|
+
ok=result.ok,
|
|
784
|
+
error=result.error,
|
|
785
|
+
data={
|
|
786
|
+
"source": result.source,
|
|
787
|
+
"summary": result.summary,
|
|
788
|
+
"payload": result.payload,
|
|
789
|
+
"repo_root": str(repo_root),
|
|
790
|
+
},
|
|
791
|
+
)
|