frontend-perception-engine 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- frontend_perception_engine-0.1.0/PKG-INFO +235 -0
- frontend_perception_engine-0.1.0/README.md +222 -0
- frontend_perception_engine-0.1.0/pyproject.toml +28 -0
- frontend_perception_engine-0.1.0/setup.cfg +4 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/PKG-INFO +235 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/SOURCES.txt +53 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/dependency_links.txt +1 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/entry_points.txt +2 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/requires.txt +7 -0
- frontend_perception_engine-0.1.0/src/frontend_perception_engine.egg-info/top_level.txt +1 -0
- frontend_perception_engine-0.1.0/src/navigation/__init__.py +1 -0
- frontend_perception_engine-0.1.0/src/navigation/browser_use/__init__.py +17 -0
- frontend_perception_engine-0.1.0/src/navigation/browser_use/agent_runner.py +165 -0
- frontend_perception_engine-0.1.0/src/navigation/browser_use/hints.py +155 -0
- frontend_perception_engine-0.1.0/src/navigation/browser_use/integration.py +46 -0
- frontend_perception_engine-0.1.0/src/navigation/browser_use/llm.py +70 -0
- frontend_perception_engine-0.1.0/src/navigation/codeGraph/__init__.py +4 -0
- frontend_perception_engine-0.1.0/src/navigation/codeGraph/crg_impl.py +170 -0
- frontend_perception_engine-0.1.0/src/navigation/codeGraph/factory.py +24 -0
- frontend_perception_engine-0.1.0/src/navigation/codeGraph/interface.py +78 -0
- frontend_perception_engine-0.1.0/src/navigation/codeGraph/null_impl.py +47 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/__init__.py +4 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/__main__.py +4 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/diff.py +28 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/envelope.py +50 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/handlers.py +791 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/instructions.py +36 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/resources.py +82 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/scan_registry.py +47 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/server.py +229 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/session_store.py +87 -0
- frontend_perception_engine-0.1.0/src/navigation/mcp/tools.py +332 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/__init__.py +76 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/artifacts.py +19 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/auth_gate.py +57 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/budget.py +55 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/cdp_hub.py +122 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/dev_insights.py +625 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/exploration.py +99 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/feature_flags.py +83 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/file_upload.py +84 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/flow_graph.py +121 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/form_probe.py +148 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/iframe_context.py +76 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/observation.py +97 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/preflight.py +91 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/rich_editors.py +90 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/route_guards.py +136 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/runner.py +138 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/scan.py +74 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/scripted_actions.py +67 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/state_manager.py +114 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/verification.py +117 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/virtual_scroll.py +84 -0
- frontend_perception_engine-0.1.0/src/navigation/perception/websocket_observer.py +79 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: frontend-perception-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Frontend perception engine with optional CRG-powered navigation hints for Browser Use.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: browser-use>=0.13.3
|
|
8
|
+
Requires-Dist: code-review-graph>=2.3.6
|
|
9
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
10
|
+
Requires-Dist: mcp>=1.0.0
|
|
11
|
+
Provides-Extra: aws
|
|
12
|
+
Requires-Dist: boto3>=1.35.0; extra == "aws"
|
|
13
|
+
|
|
14
|
+
# Frontend Perception Engine (CRG as Optional Library)
|
|
15
|
+
|
|
16
|
+
This project implements a frontend navigation layer where:
|
|
17
|
+
|
|
18
|
+
- Browser Use stays the browser automation engine.
|
|
19
|
+
- Code Review Graph (CRG) is used only as an optional knowledge source.
|
|
20
|
+
- Browser execution continues even when CRG is unavailable.
|
|
21
|
+
|
|
22
|
+
## Architecture
|
|
23
|
+
|
|
24
|
+
```text
|
|
25
|
+
Cursor / Claude
|
|
26
|
+
↓
|
|
27
|
+
Our MCP
|
|
28
|
+
↓
|
|
29
|
+
Frontend Navigation Layer
|
|
30
|
+
↓
|
|
31
|
+
Browser Use
|
|
32
|
+
↓
|
|
33
|
+
Browser
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
CRG is integrated behind `ICodeGraph`, so it can be replaced later by another backend such as `FrontendInteractionGraph` without changing Browser Use orchestration.
|
|
37
|
+
|
|
38
|
+
## Dependency Integration
|
|
39
|
+
|
|
40
|
+
CRG is integrated as a dependency (not forked, not modified):
|
|
41
|
+
|
|
42
|
+
- `code-review-graph>=2.3.6`
|
|
43
|
+
- `browser-use>=0.13.3`
|
|
44
|
+
|
|
45
|
+
Install:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install -e .
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Frontend Perception MCP (No LLM in Server)
|
|
52
|
+
|
|
53
|
+
The MCP server is deterministic runtime only (browser observation + actions + verify).
|
|
54
|
+
Your coding agent (Cursor/Claude/Codex) remains the brain.
|
|
55
|
+
|
|
56
|
+
### Install
|
|
57
|
+
|
|
58
|
+
Development install:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install -e .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Published package usage (when installed from index):
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install frontend-perception-engine
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Run MCP server
|
|
71
|
+
|
|
72
|
+
Using module entrypoint:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
python -m navigation.mcp
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Using script entrypoint:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
frontend-perception-mcp
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Using `uvx` (no local install in current environment):
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
uvx --from frontend-perception-engine frontend-perception-mcp
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Cursor MCP config
|
|
91
|
+
|
|
92
|
+
```json
|
|
93
|
+
{
|
|
94
|
+
"mcpServers": {
|
|
95
|
+
"frontend-perception": {
|
|
96
|
+
"command": "python",
|
|
97
|
+
"args": ["-m", "navigation.mcp"],
|
|
98
|
+
"env": {
|
|
99
|
+
"PYTHONPATH": "C:/Users/usman/Projects/frontend-perception-engine/src"
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Runtime prerequisites
|
|
107
|
+
|
|
108
|
+
- Start the sandbox app: `cd sandbox && npm run dev`
|
|
109
|
+
- Default URL used by tests/tools: `http://localhost:5173`
|
|
110
|
+
- No API keys are required to run the MCP path itself
|
|
111
|
+
|
|
112
|
+
## CRG Documentation and Public API Notes
|
|
113
|
+
|
|
114
|
+
The integration uses CRG public tool functions from:
|
|
115
|
+
|
|
116
|
+
- `code_review_graph.tools.build.build_or_update_graph`
|
|
117
|
+
- `code_review_graph.tools.query.query_graph`
|
|
118
|
+
- `code_review_graph.tools.query.semantic_search_nodes`
|
|
119
|
+
- `code_review_graph.tools.query.get_impact_radius`
|
|
120
|
+
- `code_review_graph.tools.query.list_graph_stats`
|
|
121
|
+
- `code_review_graph.tools.query.traverse_graph_func`
|
|
122
|
+
|
|
123
|
+
### Graph lifecycle
|
|
124
|
+
|
|
125
|
+
- Initialize graph (incremental/minimal): `build_or_update_graph(full_rebuild=False, postprocess="minimal")`
|
|
126
|
+
- Refresh graph (incremental): `build_or_update_graph(full_rebuild=False)`
|
|
127
|
+
- Rebuild graph (full): `build_or_update_graph(full_rebuild=True)`
|
|
128
|
+
|
|
129
|
+
### Incremental indexing
|
|
130
|
+
|
|
131
|
+
CRG incremental path is handled by `incremental_update` under the hood and can detect changed files from VCS (`base=HEAD~1` by default).
|
|
132
|
+
|
|
133
|
+
### Watch mode
|
|
134
|
+
|
|
135
|
+
CRG supports continuous updates via:
|
|
136
|
+
|
|
137
|
+
- CLI: `code-review-graph watch`
|
|
138
|
+
- API internals: `code_review_graph.incremental.watch` and `start_watch_thread`
|
|
139
|
+
|
|
140
|
+
This wrapper does not require watch mode, but is compatible with repositories kept fresh by CRG watch/daemon.
|
|
141
|
+
|
|
142
|
+
### Querying
|
|
143
|
+
|
|
144
|
+
- Pattern queries (neighbors/file relationships): `query_graph`
|
|
145
|
+
- Search (hybrid semantic + keyword): `semantic_search_nodes`
|
|
146
|
+
- Blast radius / route impact: `get_impact_radius`
|
|
147
|
+
- Traversal/path-like exploration: `traverse_graph_func`
|
|
148
|
+
- Stats/health: `list_graph_stats`
|
|
149
|
+
|
|
150
|
+
## Wrapper Layer
|
|
151
|
+
|
|
152
|
+
All CRG coupling is isolated in:
|
|
153
|
+
|
|
154
|
+
`src/navigation/codeGraph/`
|
|
155
|
+
|
|
156
|
+
Public contract:
|
|
157
|
+
|
|
158
|
+
- `initialize()`
|
|
159
|
+
- `refresh()`
|
|
160
|
+
- `rebuild()`
|
|
161
|
+
- `search()`
|
|
162
|
+
- `shortest_path()`
|
|
163
|
+
- `get_neighbors()`
|
|
164
|
+
- `get_component()`
|
|
165
|
+
- `get_file()`
|
|
166
|
+
- `get_route()`
|
|
167
|
+
- `query()`
|
|
168
|
+
|
|
169
|
+
Future-oriented methods are already represented on `ICodeGraph`:
|
|
170
|
+
|
|
171
|
+
- `findNavigationHint(...)` style equivalent via `find_navigation_hint(...)`
|
|
172
|
+
- `find_relevant_components(...)`
|
|
173
|
+
- `find_likely_route(...)`
|
|
174
|
+
- `find_related_files(...)`
|
|
175
|
+
- `find_button_candidates(...)`
|
|
176
|
+
- `find_component_hierarchy(...)`
|
|
177
|
+
- `find_entry_point(...)`
|
|
178
|
+
|
|
179
|
+
## Browser Use Integration
|
|
180
|
+
|
|
181
|
+
`BrowserUseNavigator` provides a lightweight dry-run timeline for tests.
|
|
182
|
+
|
|
183
|
+
`PerceptionAgentRunner` runs a **real Browser Use agent** with optional graph hints injected via `extend_system_message`. Graph output is never a mandatory stage — if CRG or AWS credentials are missing, the agent either skips hints or reports a clear error.
|
|
184
|
+
|
|
185
|
+
### Live agent (Bedrock Nova)
|
|
186
|
+
|
|
187
|
+
1. Start the sandbox:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
cd sandbox && npm run dev
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
2. Configure AWS (copy `.env.example` → `.env`):
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
AWS_ACCESS_KEY_ID=...
|
|
197
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
198
|
+
AWS_REGION=us-east-1
|
|
199
|
+
BEDROCK_MODEL=amazon.nova-pro-v1:0
|
|
200
|
+
SANDBOX_URL=http://localhost:5173
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
3. Install AWS extra and run:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
pip install -e ".[aws]"
|
|
207
|
+
python src/run_agent.py --task "Add Pulse Watch to cart and complete checkout"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Dry-run (graph hints only, no browser):
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
python src/run_agent.py --dry-run --task "Log in as admin and open admin report"
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Flags:
|
|
217
|
+
|
|
218
|
+
- `--no-graph` — disable CRG hints
|
|
219
|
+
- `--headless` — headless browser
|
|
220
|
+
- `--max-steps 25` — step limit
|
|
221
|
+
- `--url http://localhost:5174` — custom sandbox URL
|
|
222
|
+
|
|
223
|
+
## Demo
|
|
224
|
+
|
|
225
|
+
Run:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
python src/demo.py
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Expected behavior:
|
|
232
|
+
|
|
233
|
+
1. Browser Use execution starts.
|
|
234
|
+
2. Optional code graph query is attempted.
|
|
235
|
+
3. Browser Use continues regardless of query success.
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# Frontend Perception Engine (CRG as Optional Library)
|
|
2
|
+
|
|
3
|
+
This project implements a frontend navigation layer where:
|
|
4
|
+
|
|
5
|
+
- Browser Use stays the browser automation engine.
|
|
6
|
+
- Code Review Graph (CRG) is used only as an optional knowledge source.
|
|
7
|
+
- Browser execution continues even when CRG is unavailable.
|
|
8
|
+
|
|
9
|
+
## Architecture
|
|
10
|
+
|
|
11
|
+
```text
|
|
12
|
+
Cursor / Claude
|
|
13
|
+
↓
|
|
14
|
+
Our MCP
|
|
15
|
+
↓
|
|
16
|
+
Frontend Navigation Layer
|
|
17
|
+
↓
|
|
18
|
+
Browser Use
|
|
19
|
+
↓
|
|
20
|
+
Browser
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
CRG is integrated behind `ICodeGraph`, so it can be replaced later by another backend such as `FrontendInteractionGraph` without changing Browser Use orchestration.
|
|
24
|
+
|
|
25
|
+
## Dependency Integration
|
|
26
|
+
|
|
27
|
+
CRG is integrated as a dependency (not forked, not modified):
|
|
28
|
+
|
|
29
|
+
- `code-review-graph>=2.3.6`
|
|
30
|
+
- `browser-use>=0.13.3`
|
|
31
|
+
|
|
32
|
+
Install:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install -e .
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Frontend Perception MCP (No LLM in Server)
|
|
39
|
+
|
|
40
|
+
The MCP server is deterministic runtime only (browser observation + actions + verify).
|
|
41
|
+
Your coding agent (Cursor/Claude/Codex) remains the brain.
|
|
42
|
+
|
|
43
|
+
### Install
|
|
44
|
+
|
|
45
|
+
Development install:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install -e .
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Published package usage (when installed from index):
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install frontend-perception-engine
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### Run MCP server
|
|
58
|
+
|
|
59
|
+
Using module entrypoint:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m navigation.mcp
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Using script entrypoint:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
frontend-perception-mcp
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Using `uvx` (no local install in current environment):
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
uvx --from frontend-perception-engine frontend-perception-mcp
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Cursor MCP config
|
|
78
|
+
|
|
79
|
+
```json
|
|
80
|
+
{
|
|
81
|
+
"mcpServers": {
|
|
82
|
+
"frontend-perception": {
|
|
83
|
+
"command": "python",
|
|
84
|
+
"args": ["-m", "navigation.mcp"],
|
|
85
|
+
"env": {
|
|
86
|
+
"PYTHONPATH": "C:/Users/usman/Projects/frontend-perception-engine/src"
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Runtime prerequisites
|
|
94
|
+
|
|
95
|
+
- Start the sandbox app: `cd sandbox && npm run dev`
|
|
96
|
+
- Default URL used by tests/tools: `http://localhost:5173`
|
|
97
|
+
- No API keys are required to run the MCP path itself
|
|
98
|
+
|
|
99
|
+
## CRG Documentation and Public API Notes
|
|
100
|
+
|
|
101
|
+
The integration uses CRG public tool functions from:
|
|
102
|
+
|
|
103
|
+
- `code_review_graph.tools.build.build_or_update_graph`
|
|
104
|
+
- `code_review_graph.tools.query.query_graph`
|
|
105
|
+
- `code_review_graph.tools.query.semantic_search_nodes`
|
|
106
|
+
- `code_review_graph.tools.query.get_impact_radius`
|
|
107
|
+
- `code_review_graph.tools.query.list_graph_stats`
|
|
108
|
+
- `code_review_graph.tools.query.traverse_graph_func`
|
|
109
|
+
|
|
110
|
+
### Graph lifecycle
|
|
111
|
+
|
|
112
|
+
- Initialize graph (incremental/minimal): `build_or_update_graph(full_rebuild=False, postprocess="minimal")`
|
|
113
|
+
- Refresh graph (incremental): `build_or_update_graph(full_rebuild=False)`
|
|
114
|
+
- Rebuild graph (full): `build_or_update_graph(full_rebuild=True)`
|
|
115
|
+
|
|
116
|
+
### Incremental indexing
|
|
117
|
+
|
|
118
|
+
CRG incremental path is handled by `incremental_update` under the hood and can detect changed files from VCS (`base=HEAD~1` by default).
|
|
119
|
+
|
|
120
|
+
### Watch mode
|
|
121
|
+
|
|
122
|
+
CRG supports continuous updates via:
|
|
123
|
+
|
|
124
|
+
- CLI: `code-review-graph watch`
|
|
125
|
+
- API internals: `code_review_graph.incremental.watch` and `start_watch_thread`
|
|
126
|
+
|
|
127
|
+
This wrapper does not require watch mode, but is compatible with repositories kept fresh by CRG watch/daemon.
|
|
128
|
+
|
|
129
|
+
### Querying
|
|
130
|
+
|
|
131
|
+
- Pattern queries (neighbors/file relationships): `query_graph`
|
|
132
|
+
- Search (hybrid semantic + keyword): `semantic_search_nodes`
|
|
133
|
+
- Blast radius / route impact: `get_impact_radius`
|
|
134
|
+
- Traversal/path-like exploration: `traverse_graph_func`
|
|
135
|
+
- Stats/health: `list_graph_stats`
|
|
136
|
+
|
|
137
|
+
## Wrapper Layer
|
|
138
|
+
|
|
139
|
+
All CRG coupling is isolated in:
|
|
140
|
+
|
|
141
|
+
`src/navigation/codeGraph/`
|
|
142
|
+
|
|
143
|
+
Public contract:
|
|
144
|
+
|
|
145
|
+
- `initialize()`
|
|
146
|
+
- `refresh()`
|
|
147
|
+
- `rebuild()`
|
|
148
|
+
- `search()`
|
|
149
|
+
- `shortest_path()`
|
|
150
|
+
- `get_neighbors()`
|
|
151
|
+
- `get_component()`
|
|
152
|
+
- `get_file()`
|
|
153
|
+
- `get_route()`
|
|
154
|
+
- `query()`
|
|
155
|
+
|
|
156
|
+
Future-oriented methods are already represented on `ICodeGraph`:
|
|
157
|
+
|
|
158
|
+
- `findNavigationHint(...)` style equivalent via `find_navigation_hint(...)`
|
|
159
|
+
- `find_relevant_components(...)`
|
|
160
|
+
- `find_likely_route(...)`
|
|
161
|
+
- `find_related_files(...)`
|
|
162
|
+
- `find_button_candidates(...)`
|
|
163
|
+
- `find_component_hierarchy(...)`
|
|
164
|
+
- `find_entry_point(...)`
|
|
165
|
+
|
|
166
|
+
## Browser Use Integration
|
|
167
|
+
|
|
168
|
+
`BrowserUseNavigator` provides a lightweight dry-run timeline for tests.
|
|
169
|
+
|
|
170
|
+
`PerceptionAgentRunner` runs a **real Browser Use agent** with optional graph hints injected via `extend_system_message`. Graph output is never a mandatory stage — if CRG or AWS credentials are missing, the agent either skips hints or reports a clear error.
|
|
171
|
+
|
|
172
|
+
### Live agent (Bedrock Nova)
|
|
173
|
+
|
|
174
|
+
1. Start the sandbox:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
cd sandbox && npm run dev
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
2. Configure AWS (copy `.env.example` → `.env`):
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
AWS_ACCESS_KEY_ID=...
|
|
184
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
185
|
+
AWS_REGION=us-east-1
|
|
186
|
+
BEDROCK_MODEL=amazon.nova-pro-v1:0
|
|
187
|
+
SANDBOX_URL=http://localhost:5173
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
3. Install AWS extra and run:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
pip install -e ".[aws]"
|
|
194
|
+
python src/run_agent.py --task "Add Pulse Watch to cart and complete checkout"
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Dry-run (graph hints only, no browser):
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
python src/run_agent.py --dry-run --task "Log in as admin and open admin report"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Flags:
|
|
204
|
+
|
|
205
|
+
- `--no-graph` — disable CRG hints
|
|
206
|
+
- `--headless` — headless browser
|
|
207
|
+
- `--max-steps 25` — step limit
|
|
208
|
+
- `--url http://localhost:5174` — custom sandbox URL
|
|
209
|
+
|
|
210
|
+
## Demo
|
|
211
|
+
|
|
212
|
+
Run:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
python src/demo.py
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Expected behavior:
|
|
219
|
+
|
|
220
|
+
1. Browser Use execution starts.
|
|
221
|
+
2. Optional code graph query is attempted.
|
|
222
|
+
3. Browser Use continues regardless of query success.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "frontend-perception-engine"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Frontend perception engine with optional CRG-powered navigation hints for Browser Use."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"browser-use>=0.13.3",
|
|
9
|
+
"code-review-graph>=2.3.6",
|
|
10
|
+
"python-dotenv>=1.0.0",
|
|
11
|
+
"mcp>=1.0.0",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[project.scripts]
|
|
15
|
+
frontend-perception-mcp = "navigation.mcp.server:main"
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
aws = ["boto3>=1.35.0"]
|
|
19
|
+
|
|
20
|
+
[build-system]
|
|
21
|
+
requires = ["setuptools>=68", "wheel"]
|
|
22
|
+
build-backend = "setuptools.build_meta"
|
|
23
|
+
|
|
24
|
+
[tool.setuptools]
|
|
25
|
+
package-dir = {"" = "src"}
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
where = ["src"]
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: frontend-perception-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Frontend perception engine with optional CRG-powered navigation hints for Browser Use.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: browser-use>=0.13.3
|
|
8
|
+
Requires-Dist: code-review-graph>=2.3.6
|
|
9
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
10
|
+
Requires-Dist: mcp>=1.0.0
|
|
11
|
+
Provides-Extra: aws
|
|
12
|
+
Requires-Dist: boto3>=1.35.0; extra == "aws"
|
|
13
|
+
|
|
14
|
+
# Frontend Perception Engine (CRG as Optional Library)
|
|
15
|
+
|
|
16
|
+
This project implements a frontend navigation layer where:
|
|
17
|
+
|
|
18
|
+
- Browser Use stays the browser automation engine.
|
|
19
|
+
- Code Review Graph (CRG) is used only as an optional knowledge source.
|
|
20
|
+
- Browser execution continues even when CRG is unavailable.
|
|
21
|
+
|
|
22
|
+
## Architecture
|
|
23
|
+
|
|
24
|
+
```text
|
|
25
|
+
Cursor / Claude
|
|
26
|
+
↓
|
|
27
|
+
Our MCP
|
|
28
|
+
↓
|
|
29
|
+
Frontend Navigation Layer
|
|
30
|
+
↓
|
|
31
|
+
Browser Use
|
|
32
|
+
↓
|
|
33
|
+
Browser
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
CRG is integrated behind `ICodeGraph`, so it can be replaced later by another backend such as `FrontendInteractionGraph` without changing Browser Use orchestration.
|
|
37
|
+
|
|
38
|
+
## Dependency Integration
|
|
39
|
+
|
|
40
|
+
CRG is integrated as a dependency (not forked, not modified):
|
|
41
|
+
|
|
42
|
+
- `code-review-graph>=2.3.6`
|
|
43
|
+
- `browser-use>=0.13.3`
|
|
44
|
+
|
|
45
|
+
Install:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install -e .
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Frontend Perception MCP (No LLM in Server)
|
|
52
|
+
|
|
53
|
+
The MCP server is deterministic runtime only (browser observation + actions + verify).
|
|
54
|
+
Your coding agent (Cursor/Claude/Codex) remains the brain.
|
|
55
|
+
|
|
56
|
+
### Install
|
|
57
|
+
|
|
58
|
+
Development install:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install -e .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Published package usage (when installed from index):
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install frontend-perception-engine
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Run MCP server
|
|
71
|
+
|
|
72
|
+
Using module entrypoint:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
python -m navigation.mcp
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Using script entrypoint:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
frontend-perception-mcp
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Using `uvx` (no local install in current environment):
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
uvx --from frontend-perception-engine frontend-perception-mcp
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Cursor MCP config
|
|
91
|
+
|
|
92
|
+
```json
|
|
93
|
+
{
|
|
94
|
+
"mcpServers": {
|
|
95
|
+
"frontend-perception": {
|
|
96
|
+
"command": "python",
|
|
97
|
+
"args": ["-m", "navigation.mcp"],
|
|
98
|
+
"env": {
|
|
99
|
+
"PYTHONPATH": "C:/Users/usman/Projects/frontend-perception-engine/src"
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### Runtime prerequisites
|
|
107
|
+
|
|
108
|
+
- Start the sandbox app: `cd sandbox && npm run dev`
|
|
109
|
+
- Default URL used by tests/tools: `http://localhost:5173`
|
|
110
|
+
- No API keys are required to run the MCP path itself
|
|
111
|
+
|
|
112
|
+
## CRG Documentation and Public API Notes
|
|
113
|
+
|
|
114
|
+
The integration uses CRG public tool functions from:
|
|
115
|
+
|
|
116
|
+
- `code_review_graph.tools.build.build_or_update_graph`
|
|
117
|
+
- `code_review_graph.tools.query.query_graph`
|
|
118
|
+
- `code_review_graph.tools.query.semantic_search_nodes`
|
|
119
|
+
- `code_review_graph.tools.query.get_impact_radius`
|
|
120
|
+
- `code_review_graph.tools.query.list_graph_stats`
|
|
121
|
+
- `code_review_graph.tools.query.traverse_graph_func`
|
|
122
|
+
|
|
123
|
+
### Graph lifecycle
|
|
124
|
+
|
|
125
|
+
- Initialize graph (incremental/minimal): `build_or_update_graph(full_rebuild=False, postprocess="minimal")`
|
|
126
|
+
- Refresh graph (incremental): `build_or_update_graph(full_rebuild=False)`
|
|
127
|
+
- Rebuild graph (full): `build_or_update_graph(full_rebuild=True)`
|
|
128
|
+
|
|
129
|
+
### Incremental indexing
|
|
130
|
+
|
|
131
|
+
CRG incremental path is handled by `incremental_update` under the hood and can detect changed files from VCS (`base=HEAD~1` by default).
|
|
132
|
+
|
|
133
|
+
### Watch mode
|
|
134
|
+
|
|
135
|
+
CRG supports continuous updates via:
|
|
136
|
+
|
|
137
|
+
- CLI: `code-review-graph watch`
|
|
138
|
+
- API internals: `code_review_graph.incremental.watch` and `start_watch_thread`
|
|
139
|
+
|
|
140
|
+
This wrapper does not require watch mode, but is compatible with repositories kept fresh by CRG watch/daemon.
|
|
141
|
+
|
|
142
|
+
### Querying
|
|
143
|
+
|
|
144
|
+
- Pattern queries (neighbors/file relationships): `query_graph`
|
|
145
|
+
- Search (hybrid semantic + keyword): `semantic_search_nodes`
|
|
146
|
+
- Blast radius / route impact: `get_impact_radius`
|
|
147
|
+
- Traversal/path-like exploration: `traverse_graph_func`
|
|
148
|
+
- Stats/health: `list_graph_stats`
|
|
149
|
+
|
|
150
|
+
## Wrapper Layer
|
|
151
|
+
|
|
152
|
+
All CRG coupling is isolated in:
|
|
153
|
+
|
|
154
|
+
`src/navigation/codeGraph/`
|
|
155
|
+
|
|
156
|
+
Public contract:
|
|
157
|
+
|
|
158
|
+
- `initialize()`
|
|
159
|
+
- `refresh()`
|
|
160
|
+
- `rebuild()`
|
|
161
|
+
- `search()`
|
|
162
|
+
- `shortest_path()`
|
|
163
|
+
- `get_neighbors()`
|
|
164
|
+
- `get_component()`
|
|
165
|
+
- `get_file()`
|
|
166
|
+
- `get_route()`
|
|
167
|
+
- `query()`
|
|
168
|
+
|
|
169
|
+
Future-oriented methods are already represented on `ICodeGraph`:
|
|
170
|
+
|
|
171
|
+
- `findNavigationHint(...)` style equivalent via `find_navigation_hint(...)`
|
|
172
|
+
- `find_relevant_components(...)`
|
|
173
|
+
- `find_likely_route(...)`
|
|
174
|
+
- `find_related_files(...)`
|
|
175
|
+
- `find_button_candidates(...)`
|
|
176
|
+
- `find_component_hierarchy(...)`
|
|
177
|
+
- `find_entry_point(...)`
|
|
178
|
+
|
|
179
|
+
## Browser Use Integration
|
|
180
|
+
|
|
181
|
+
`BrowserUseNavigator` provides a lightweight dry-run timeline for tests.
|
|
182
|
+
|
|
183
|
+
`PerceptionAgentRunner` runs a **real Browser Use agent** with optional graph hints injected via `extend_system_message`. Graph output is never a mandatory stage — if CRG or AWS credentials are missing, the agent either skips hints or reports a clear error.
|
|
184
|
+
|
|
185
|
+
### Live agent (Bedrock Nova)
|
|
186
|
+
|
|
187
|
+
1. Start the sandbox:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
cd sandbox && npm run dev
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
2. Configure AWS (copy `.env.example` → `.env`):
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
AWS_ACCESS_KEY_ID=...
|
|
197
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
198
|
+
AWS_REGION=us-east-1
|
|
199
|
+
BEDROCK_MODEL=amazon.nova-pro-v1:0
|
|
200
|
+
SANDBOX_URL=http://localhost:5173
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
3. Install AWS extra and run:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
pip install -e ".[aws]"
|
|
207
|
+
python src/run_agent.py --task "Add Pulse Watch to cart and complete checkout"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Dry-run (graph hints only, no browser):
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
python src/run_agent.py --dry-run --task "Log in as admin and open admin report"
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
Flags:
|
|
217
|
+
|
|
218
|
+
- `--no-graph` — disable CRG hints
|
|
219
|
+
- `--headless` — headless browser
|
|
220
|
+
- `--max-steps 25` — step limit
|
|
221
|
+
- `--url http://localhost:5174` — custom sandbox URL
|
|
222
|
+
|
|
223
|
+
## Demo
|
|
224
|
+
|
|
225
|
+
Run:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
python src/demo.py
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Expected behavior:
|
|
232
|
+
|
|
233
|
+
1. Browser Use execution starts.
|
|
234
|
+
2. Optional code graph query is attempted.
|
|
235
|
+
3. Browser Use continues regardless of query success.
|