jev-browser-pilot 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jev_browser_pilot-0.1.1/LICENSE +21 -0
- jev_browser_pilot-0.1.1/PKG-INFO +483 -0
- jev_browser_pilot-0.1.1/README.md +457 -0
- jev_browser_pilot-0.1.1/pyproject.toml +54 -0
- jev_browser_pilot-0.1.1/setup.cfg +4 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/PKG-INFO +483 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/SOURCES.txt +41 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/dependency_links.txt +1 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/entry_points.txt +2 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/requires.txt +10 -0
- jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/top_level.txt +1 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/__init__.py +66 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/bench.py +178 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/browser.py +298 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/cli.py +554 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/desktop.py +234 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/loop.py +214 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/perception.py +275 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/policy.py +100 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/providers/__init__.py +48 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/providers/base.py +147 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/providers/jev.py +116 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/providers/openai_compat.py +164 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/safety.py +99 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/serialize.py +91 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/surface.py +71 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/trace.py +203 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/types.py +181 -0
- jev_browser_pilot-0.1.1/src/jev_pilot/verify.py +169 -0
- jev_browser_pilot-0.1.1/tests/test_bench.py +77 -0
- jev_browser_pilot-0.1.1/tests/test_browser.py +99 -0
- jev_browser_pilot-0.1.1/tests/test_cli.py +236 -0
- jev_browser_pilot-0.1.1/tests/test_desktop.py +63 -0
- jev_browser_pilot-0.1.1/tests/test_desktop_loop.py +146 -0
- jev_browser_pilot-0.1.1/tests/test_jev_provider.py +99 -0
- jev_browser_pilot-0.1.1/tests/test_loop.py +159 -0
- jev_browser_pilot-0.1.1/tests/test_perception.py +78 -0
- jev_browser_pilot-0.1.1/tests/test_policy.py +80 -0
- jev_browser_pilot-0.1.1/tests/test_providers.py +161 -0
- jev_browser_pilot-0.1.1/tests/test_safety.py +86 -0
- jev_browser_pilot-0.1.1/tests/test_serialize.py +72 -0
- jev_browser_pilot-0.1.1/tests/test_trace.py +75 -0
- jev_browser_pilot-0.1.1/tests/test_verify.py +66 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aidil Zafri
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jev-browser-pilot
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: A bounded, cheap decision layer for browser and desktop automation: a System One model picks the next step, your code owns observation, content, actuation and verification.
|
|
5
|
+
Author-email: Aidil Zafri <aidilzafri56@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/aidil2105/jev-browser-pilot
|
|
8
|
+
Project-URL: Issues, https://github.com/aidil2105/jev-browser-pilot/issues
|
|
9
|
+
Keywords: computer-use,browser-automation,agents,decision-model,jev,typesafe,cdp
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: websocket-client>=1.6
|
|
20
|
+
Provides-Extra: jev
|
|
21
|
+
Requires-Dist: typesafe-sdk>=0.6; python_version >= "3.10" and extra == "jev"
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
24
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# jev-browser-pilot
|
|
28
|
+
|
|
29
|
+
[](https://github.com/aidil2105/jev-browser-pilot/actions/workflows/ci.yml)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
[](pyproject.toml)
|
|
32
|
+
|
|
33
|
+
A bounded decision layer for browser and desktop automation. Your code observes the surface,
|
|
34
|
+
builds the list of things it could do, executes one of them, and checks the result. A
|
|
35
|
+
decision-only model makes exactly one choice per step: which candidate is next.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
observe (your code) -> decide (the model, one typed choice) -> act (your code) -> verify (your code)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The loop is not new. What is different is the split. The model never sees a screenshot, never
|
|
42
|
+
writes a selector, never produces text to type, and is never asked whether it succeeded. It
|
|
43
|
+
answers one bounded question over a list of ids your code built, and returns a confidence your
|
|
44
|
+
code routes on. Everything that can fail silently stays in code.
|
|
45
|
+
|
|
46
|
+
That buys two things: a step that costs a fraction of a cent and answers in a few hundred
|
|
47
|
+
milliseconds, and an audit trail that shows exactly what the model was shown at every step.
|
|
48
|
+
|
|
49
|
+
## Install
|
|
50
|
+
|
|
51
|
+
Not on PyPI yet, so `pip install jev-browser-pilot` does not resolve. Use the repository:
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
pip install "git+https://github.com/aidil2105/jev-browser-pilot"
|
|
55
|
+
# with the Jev provider (Python 3.10 or newer):
|
|
56
|
+
pip install "jev-browser-pilot[jev] @ git+https://github.com/aidil2105/jev-browser-pilot"
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Or from a clone, which is what the commands below assume:
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
git clone https://github.com/aidil2105/jev-browser-pilot
|
|
63
|
+
cd jev-browser-pilot
|
|
64
|
+
python -m venv .venv
|
|
65
|
+
.venv/Scripts/python -m pip install -e ".[dev,jev]" # Windows
|
|
66
|
+
# .venv/bin/python -m pip install -e ".[dev,jev]" # macOS and Linux
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The only runtime dependency is `websocket-client`. The browser surface uses the Chrome,
|
|
70
|
+
Chromium or Edge you already have. The Jev provider needs Python 3.10 or newer because the
|
|
71
|
+
vendor SDK does; on 3.9 the core, the OpenAI-compatible provider and the mock chooser all work
|
|
72
|
+
and the Jev provider says so plainly instead of failing to install.
|
|
73
|
+
|
|
74
|
+
Releases are built and uploaded by `.github/workflows/publish.yml` when a GitHub release is
|
|
75
|
+
published, using PyPI's trusted publishing rather than a stored token; `docs/publishing.md` has
|
|
76
|
+
the one-time PyPI setup and the local fallback.
|
|
77
|
+
|
|
78
|
+
## Quickstart
|
|
79
|
+
|
|
80
|
+
Three commands, in this order. The first one needs nothing at all.
|
|
81
|
+
|
|
82
|
+
**1. Prove the loop works, offline.** No browser, no network, no credentials:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
$ jev-pilot selftest
|
|
86
|
+
jev-browser-pilot 0.1.0 selftest
|
|
87
|
+
step 1: act pick=2 link 'Omega appendix' 0ms
|
|
88
|
+
pass: url-contains:omega.html
|
|
89
|
+
pass: text-contains:goal reached
|
|
90
|
+
reached=True stopped=reached steps=1 actions=['2']
|
|
91
|
+
pass: unknown id is an error
|
|
92
|
+
pass: low confidence escalates with the pick
|
|
93
|
+
pass: transport failure is an error
|
|
94
|
+
selftest: PASS
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**2. Borrow just the decision step.** JSON in, JSON out, so any harness can use it without
|
|
98
|
+
adopting the loop:
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
$ echo '{"goal":"open the Omega appendix",
|
|
102
|
+
"state":"CANDIDATE ACTIONS:\n[1] link (Alpha report)\n[2] link (Omega appendix)",
|
|
103
|
+
"options":[{"id":"1","label":"link (Alpha report)"},
|
|
104
|
+
{"id":"2","label":"link (Omega appendix)"}]}' \
|
|
105
|
+
| jev-pilot decide --provider mock
|
|
106
|
+
{
|
|
107
|
+
"outcome": "act",
|
|
108
|
+
"pick": "2",
|
|
109
|
+
"label": "link (Omega appendix)",
|
|
110
|
+
"confidence": 0.8,
|
|
111
|
+
"provider": "keyword",
|
|
112
|
+
"reason": "picked"
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Output trimmed to the fields that matter; the real object also carries the full probability map,
|
|
117
|
+
tokens, cost and latency. The mock provider reports itself as `keyword`, which is what it is: a
|
|
118
|
+
deterministic keyword overlap, so the quickstart needs no key and no network.
|
|
119
|
+
|
|
120
|
+
**3. Drive a real site.** The browser gets its own profile and its own debugging port, and the
|
|
121
|
+
hosts it may touch are declared up front:
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
$ jev-pilot run --start "https://en.wikipedia.org/wiki/Calculator" \
|
|
125
|
+
--task-file examples/wikipedia-chain.json \
|
|
126
|
+
--provider jev --headless --trace run.jsonl --report run.html
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Exit codes are part of the contract, because this is meant to run in a pipeline: `0` every goal
|
|
130
|
+
reached its postcondition, `1` a goal did not reach it, `2` a hard error (no browser, no
|
|
131
|
+
credentials, a safety refusal, bad input).
|
|
132
|
+
|
|
133
|
+
## What a run looks like
|
|
134
|
+
|
|
135
|
+
Real output, three chained goals in one browser session, one decision per hop:
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
jev-browser-pilot 0.1.0
|
|
139
|
+
surface: browser | chooser: jev model=jev-latest
|
|
140
|
+
safety: hosts=[en.wikipedia.org] actions=['click'] typing=off dry_run=False max_steps=12
|
|
141
|
+
session: chrome pid=9804 port=33154 profile=33154
|
|
142
|
+
|
|
143
|
+
=== goal: open the article about the slide rule
|
|
144
|
+
step 1: act conf=0.95 pick=100 a 'slide rule' 1454ms
|
|
145
|
+
pass: url-contains:Slide_rule ('Slide_rule' in 'https://en.wikipedia.org/wiki/Slide_rule')
|
|
146
|
+
result: reached=True stopped=reached steps=1 median=1454ms cost=$0.000270
|
|
147
|
+
|
|
148
|
+
=== goal: open the article about William Oughtred, the inventor of the slide rule
|
|
149
|
+
step 1: act conf=0.99 pick=14 a 'William Oughtred' 297ms
|
|
150
|
+
pass: url-contains:William_Oughtred ('William_Oughtred' in 'https://en.wikipedia.org/wiki/William_Oughtred')
|
|
151
|
+
result: reached=True stopped=reached steps=1 median=297ms cost=$0.000286
|
|
152
|
+
|
|
153
|
+
=== goal: open the article about Richard Delamain, the rival who claimed priority for the slide rule
|
|
154
|
+
step 1: act conf=0.47 pick=88 a 'Richard Delamain' 295ms
|
|
155
|
+
pass: url-contains:Delamain ('Delamain' in 'https://en.wikipedia.org/wiki/Richard_Delamaine')
|
|
156
|
+
result: reached=True stopped=reached steps=1 median=295ms cost=$0.000301
|
|
157
|
+
report: run.html
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Three goals, three decisions, $0.00086 for the session. Note the third hop: it is the only one
|
|
161
|
+
whose goal describes a person rather than nearly quoting the link text, and it is the only one
|
|
162
|
+
answered at 0.47 confidence. Right answer, visibly less sure. That confidence is the signal you
|
|
163
|
+
route on.
|
|
164
|
+
## Why this shape
|
|
165
|
+
|
|
166
|
+
An agent that reads a page with a full language model pays seconds and cents per step for a
|
|
167
|
+
judgment that is often a single pick among a handful of visible options. Measured on five frozen
|
|
168
|
+
observations of a real window, three repeats each, no capture and no actuation involved, so this is
|
|
169
|
+
the decision alone (`examples/bench-calculator.json`):
|
|
170
|
+
|
|
171
|
+
| chooser | correct | accuracy | answered | median | range | cost |
|
|
172
|
+
|---|---|---|---|---|---|---|
|
|
173
|
+
| `jev` | 15/15 | 100% | 15/15 | 349 ms | 277 to 870 ms | $0.000393 |
|
|
174
|
+
| `openai`, a mid-tier chat model | 12/15 | 80% | 12/15 | 2,858 ms | 4 to 15,328 ms | not metered here |
|
|
175
|
+
| `mock`, keyword overlap | 3/15 | 20% | 15/15 | 0 ms | 0 ms | free |
|
|
176
|
+
|
|
177
|
+
Read that honestly: the chat model's three misses are all the same transport failure on the hardest
|
|
178
|
+
state (a relay 503, not a wrong answer), so the column that matters most between it and `jev` here
|
|
179
|
+
is latency. `jev` answered every call, never wrong, at 0.93 to 1.00 confidence on the four
|
|
180
|
+
deterministic states and 0.78 on the one that relies on reading a display. The `mock` chooser fails
|
|
181
|
+
every state whose answer is not in the goal's own words, which is exactly what a naive keyword
|
|
182
|
+
overlap should do.
|
|
183
|
+
|
|
184
|
+
Reproduce it, or run it against your own states:
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
jev-pilot bench --fixtures examples/bench-calculator.json \
|
|
188
|
+
--chooser jev --chooser mock --chooser openai:llama3.1 \
|
|
189
|
+
--base-url http://localhost:11434/v1 --repeats 3
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The `mock` chooser is a deterministic keyword overlap stand-in, deliberately naive: it exists so
|
|
193
|
+
the library, its tests and its docs work with no key, no network and no spend. It is a fixture,
|
|
194
|
+
not a baseline to beat.
|
|
195
|
+
|
|
196
|
+
## What this is not
|
|
197
|
+
|
|
198
|
+
- **Not an autonomous agent.** There is no lookahead. If the next step is not derivable from the
|
|
199
|
+
current observation, the model answers `__stuck__` instead of guessing. Put a planner above it.
|
|
200
|
+
- **Not a text generator.** It cannot produce the string to type, the URL to open or the value to
|
|
201
|
+
set. Your code owns content. It also means no prompt injection can make it invent one.
|
|
202
|
+
- **Not a success oracle.** Postconditions are evaluated by code, before and after every action.
|
|
203
|
+
The model's `done` is a report, not a proof.
|
|
204
|
+
|
|
205
|
+
## How it fits together
|
|
206
|
+
|
|
207
|
+
| piece | what it does |
|
|
208
|
+
|---|---|
|
|
209
|
+
| `Surface` | whatever you are driving: `BrowserPilot`, `DesktopPilot`, `ScriptedSurface`, or your own object with `observe`/`act`/`url`/`probe` |
|
|
210
|
+
| `perception` | turns a page or a window into `Element`s and mints the candidate `Option` ids |
|
|
211
|
+
| `serialize` | renders the exact state string the model sees |
|
|
212
|
+
| `Chooser` | the decision backend: `jev`, `openai` (any compatible endpoint), `mock`, `scripted`, or yours |
|
|
213
|
+
| `policy` | what an answer is allowed to mean, in code rather than in a prompt |
|
|
214
|
+
| `verify` | deterministic postconditions: URL, text, CSS selector, file contents |
|
|
215
|
+
| `safety` | declared hosts, allowed actions, typing off by default, dry run |
|
|
216
|
+
| `trace` | JSONL per step plus a self-contained HTML report |
|
|
217
|
+
| `bench` | the same frozen states replayed against several choosers |
|
|
218
|
+
|
|
219
|
+
The library API is small enough to quote:
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
from jev_pilot import SafetyPolicy, run_episode
|
|
223
|
+
from jev_pilot.browser import BrowserPilot
|
|
224
|
+
from jev_pilot.providers import build_chooser
|
|
225
|
+
|
|
226
|
+
safety = SafetyPolicy.for_url("https://en.wikipedia.org/wiki/Calculator")
|
|
227
|
+
chooser = build_chooser("jev") # or "openai" with model=..., or "mock"
|
|
228
|
+
|
|
229
|
+
with BrowserPilot("https://en.wikipedia.org/wiki/Calculator", headless=True,
|
|
230
|
+
safety=safety) as pilot:
|
|
231
|
+
episode = run_episode(
|
|
232
|
+
pilot, chooser,
|
|
233
|
+
goal="open the article about the slide rule",
|
|
234
|
+
verifiers=["url-contains:Slide_rule"],
|
|
235
|
+
steps=3, floor=0.6, safety=safety,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
print(episode.reached, episode.stopped, episode.summary())
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## The decision policy
|
|
242
|
+
|
|
243
|
+
One question per step: a single choice over the candidate ids, plus two sentinels. The rules that
|
|
244
|
+
turn an answer into an outcome live in `policy.py`:
|
|
245
|
+
|
|
246
|
+
| outcome | meaning | who handles it |
|
|
247
|
+
|---|---|---|
|
|
248
|
+
| `act` | a candidate was chosen and is safe to execute | the loop clicks it |
|
|
249
|
+
| `done` | the model reports the goal is achieved | the loop stops; the postcondition still decides `reached` |
|
|
250
|
+
| `stuck` | nothing in the list can make progress | the loop stops; a planner above decides what next |
|
|
251
|
+
| `escalate` | the pick is under the confidence floor, **with the pick attached** | your planner, or a human |
|
|
252
|
+
| `error` | unknown id, empty answer, transport failure | your caller; never treated as a pick |
|
|
253
|
+
|
|
254
|
+
Where an episode can stop is equally explicit: `reached`, `done`, `stuck`, `escalate`, `error`,
|
|
255
|
+
`blocked` (the surface left the declared hosts), `no_progress` (the same state twice in a row),
|
|
256
|
+
`budget`, `dry_run`.
|
|
257
|
+
|
|
258
|
+
Asking one question instead of two is deliberate: an earlier version asked for the action and the
|
|
259
|
+
target separately and produced contradictory answers, picking an element while also declaring the
|
|
260
|
+
task done. One question, one answer, one decision.
|
|
261
|
+
|
|
262
|
+
## Providers, including yours
|
|
263
|
+
|
|
264
|
+
| name | what it is | notes |
|
|
265
|
+
|---|---|---|
|
|
266
|
+
| `jev` | TypeSafe Jev, a decision-only model | needs `TYPESAFE_API_KEY`; returns the full probability map |
|
|
267
|
+
| `openai` | any OpenAI-compatible endpoint | Ollama, vLLM, LM Studio, a gateway; needs `--model`, no confidence returned |
|
|
268
|
+
| `mock` | deterministic keyword overlap | credential-free, for tests, docs and offline development |
|
|
269
|
+
| `scripted` | a fixed sequence of answers | assertions in tests |
|
|
270
|
+
|
|
271
|
+
A chooser is one method, so adding yours does not touch the loop, the safety rails or the
|
|
272
|
+
verification:
|
|
273
|
+
|
|
274
|
+
```python
|
|
275
|
+
from jev_pilot.providers.base import Chooser, RawAnswer
|
|
276
|
+
|
|
277
|
+
class MyChooser(Chooser):
|
|
278
|
+
name = "mine"
|
|
279
|
+
|
|
280
|
+
def choose(self, *, goal, state, options, timeout=None):
|
|
281
|
+
answer = my_service(goal, state, [o.as_dict() for o in options])
|
|
282
|
+
return RawAnswer(pick=answer["id"], confidence=answer.get("confidence"))
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
`RawAnswer` is not permission to act. It goes through `apply_policy`, which is where the floor,
|
|
286
|
+
the sentinels and the fail-closed rules apply to every provider equally.
|
|
287
|
+
|
|
288
|
+
## Postconditions
|
|
289
|
+
|
|
290
|
+
Verification is a list of specs, checked before the first decision and after every action. The
|
|
291
|
+
success decision never involves the model.
|
|
292
|
+
|
|
293
|
+
| spec | passes when |
|
|
294
|
+
|---|---|
|
|
295
|
+
| `url-contains:TEXT` | the current URL contains TEXT (case sensitive) |
|
|
296
|
+
| `url-regex:PATTERN` | the URL matches the pattern |
|
|
297
|
+
| `text-contains:TEXT` | the visible text contains TEXT (case and whitespace insensitive) |
|
|
298
|
+
| `selector:CSS` | the page has at least one matching element |
|
|
299
|
+
| `file-contains:PATH::TEXT` | the file exists and contains TEXT |
|
|
300
|
+
|
|
301
|
+
Prefer a postcondition the page cannot fake. `text-contains:` is the weakest of them, because
|
|
302
|
+
marketing copy that merely mentions the word passes it. A minted order id, a confirmation element
|
|
303
|
+
or a file that a download had to produce are all stronger.
|
|
304
|
+
## Safety rails
|
|
305
|
+
|
|
306
|
+
Opt-in by construction, because this drives a real browser on a real machine.
|
|
307
|
+
|
|
308
|
+
- **Hosts are declared, not inferred.** Navigation is confined to the hosts you name (a `file://`
|
|
309
|
+
page reports the pseudo-host `file`). The moment an observation lands somewhere else, the
|
|
310
|
+
episode stops with `blocked`, that outranks a postcondition which would have passed there, and a
|
|
311
|
+
chain abandons its remaining tasks.
|
|
312
|
+
- **Only `click` is allowed** by default. Typing is off until you enable it, and there is no API in
|
|
313
|
+
this library for passwords, tokens or card numbers at all.
|
|
314
|
+
- **The browser is isolated.** Its own `--user-data-dir` and its own debugging port, so it cannot
|
|
315
|
+
attach to a logged-in profile.
|
|
316
|
+
- **`--dry-run` decides and records, and stops before anything is actuated.** Useful when wiring a
|
|
317
|
+
new surface, and useful for showing what the model would have done.
|
|
318
|
+
|
|
319
|
+
The rails are checked by the loop, not requested of the model, so a provider that hallucinates a
|
|
320
|
+
better plan cannot walk around them.
|
|
321
|
+
|
|
322
|
+
## Benchmarks over frozen states
|
|
323
|
+
|
|
324
|
+
A live comparison mostly measures who got luckier with perception. `bench` removes that variable:
|
|
325
|
+
one observation, frozen to a file, replayed against several choosers, scored against a known
|
|
326
|
+
answer.
|
|
327
|
+
|
|
328
|
+
```json
|
|
329
|
+
{"cases": [
|
|
330
|
+
{"name": "after-plus",
|
|
331
|
+
"category": "calculator",
|
|
332
|
+
"goal": "compute 5 + 3, stopping when the display shows the answer",
|
|
333
|
+
"state": "DISPLAY: 'Display is 5'\nACTIONS ALREADY TAKEN THIS EPISODE: Five, Plus\n[23] Button 'Plus'\n[24] Button 'Equals'\n[28] Button 'Three'",
|
|
334
|
+
"options": [{"id": "23", "label": "Button 'Plus'"},
|
|
335
|
+
{"id": "24", "label": "Button 'Equals'"},
|
|
336
|
+
{"id": "28", "label": "Button 'Three'"}],
|
|
337
|
+
"expect": "Button 'Three'"}
|
|
338
|
+
]}
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
`expect` is a label, an `id:`, or a sentinel (`__done__`, `__stuck__`). The report gives you
|
|
342
|
+
accuracy next to **coverage**, because a chooser that answers 6 of 10 calls and gets them right is
|
|
343
|
+
not the same as one that answers all 10.
|
|
344
|
+
|
|
345
|
+
```
|
|
346
|
+
jev-pilot bench --fixtures examples/bench-calculator.json --chooser jev --repeats 3 \
|
|
347
|
+
--markdown bench.md --out bench.json
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
### The same tasks, two choosers, twice each
|
|
351
|
+
|
|
352
|
+
The frozen states above isolate a single decision. This runs the whole loop instead: one task
|
|
353
|
+
file, one start page, the same postconditions, one arm per chooser, two passes, headless Chrome.
|
|
354
|
+
|
|
355
|
+
```
|
|
356
|
+
jev-pilot run --start https://en.wikipedia.org/wiki/Calculator \
|
|
357
|
+
--task-file examples/wikipedia-chain.json --provider jev --headless
|
|
358
|
+
|
|
359
|
+
jev-pilot run --start https://en.wikipedia.org/wiki/Calculator \
|
|
360
|
+
--task-file examples/wikipedia-chain.json --provider openai \
|
|
361
|
+
--model <a free open-weights chat model> --base-url <an OpenAI-compatible relay> --headless
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
| chooser | tasks reached | per-decision latency | per-decision cost |
|
|
365
|
+
|---|---|---|---|
|
|
366
|
+
| `jev` | 6 of 6 | 302 to 982 ms | $0.00027 to $0.00030 |
|
|
367
|
+
| a free open-weights chat model | 4 of 6 | 2,733 to 51,665 ms | unmetered, free route |
|
|
368
|
+
|
|
369
|
+
The four run summaries behind this table are committed under `docs/evidence/live-comparison/`, so
|
|
370
|
+
the numbers can be read back instead of taken on trust.
|
|
371
|
+
|
|
372
|
+
What this does and does not say. Whenever the chat model answered it picked the same element Jev
|
|
373
|
+
picked, every time, so the two agreed on all four decisions it completed. Its two failures were
|
|
374
|
+
not wrong picks: they were calls that never returned, both on the same task, at 30 s and 52 s,
|
|
375
|
+
from a relay that answered with an empty response where Jev answered in 302 ms at 0.49
|
|
376
|
+
confidence. Read it as a reliability and latency result under this harness, not as a ranking of
|
|
377
|
+
two models' judgment. Three tasks, one site, two passes: enough to show the shape of the
|
|
378
|
+
difference, not enough to generalise past it.
|
|
379
|
+
|
|
380
|
+
It also shows the exit codes doing their job: the arm that missed a task exited 1, the arm that
|
|
381
|
+
reached all three exited 0.
|
|
382
|
+
|
|
383
|
+
## Exit codes
|
|
384
|
+
|
|
385
|
+
Running `run` from a script is the point, so the codes are part of the interface.
|
|
386
|
+
|
|
387
|
+
| code | meaning |
|
|
388
|
+
|---|---|
|
|
389
|
+
| 0 | every task reached its postcondition |
|
|
390
|
+
| 1 | at least one task was not reached: the postcondition failed, the chooser never answered, or the episode stopped `blocked` because the surface left the declared hosts |
|
|
391
|
+
| 2 | a usage or configuration problem: an unknown chooser, an unreadable fixture, a missing credential |
|
|
392
|
+
|
|
393
|
+
Confidence never turns a miss into a success. An episode where every decision was confident and
|
|
394
|
+
the postcondition still failed exits 1, and the run prints which task it was.
|
|
395
|
+
|
|
396
|
+
## Traces and reports
|
|
397
|
+
|
|
398
|
+
Every step is recorded: the exact state sent, the ids offered, the answer, the confidence, the
|
|
399
|
+
latency, the tokens and the cost. `jev-pilot report` renders it into a single self-contained HTML
|
|
400
|
+
file with no external assets, so it can be attached to a bug report or a pull request.
|
|
401
|
+
|
|
402
|
+
```
|
|
403
|
+
jev-pilot run ... --trace run.jsonl --report run.html
|
|
404
|
+
jev-pilot report --trace run.jsonl --out run.html # render later
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
Use `--no-verbatim-state` when the page content must not be persisted. The decision is still
|
|
408
|
+
recorded; only the state text is left out.
|
|
409
|
+
|
|
410
|
+
## Known limits
|
|
411
|
+
|
|
412
|
+
- **Perception is the ceiling.** On a Wikipedia article, 200 of 447 candidate links fit the cap.
|
|
413
|
+
What falls past it is invisible, so a correct `__stuck__` can mean "I cannot see it" rather than
|
|
414
|
+
"it is not there". Check `matched_total` in the trace before blaming the model.
|
|
415
|
+
- **Confidence is a router, not a gate you set once.** The same ambiguous hop scored 0.47 in one
|
|
416
|
+
run, 0.49 in another and 0.61 in two more. Set the floor from the distribution you observe on
|
|
417
|
+
your own task.
|
|
418
|
+
- **No lookahead, no typing, no re-planning.** Those belong to the layer above.
|
|
419
|
+
- **The desktop surface is experimental.** It drives a native window through a UI Automation tree
|
|
420
|
+
via cua-driver. It has completed a real four-click episode (see below), but UIA quality varies
|
|
421
|
+
by application and it has had far less use than the browser surface.
|
|
422
|
+
- **No screenshot perception.** Text states only, by design: cheaper, auditable and diffable.
|
|
423
|
+
|
|
424
|
+
## What has been verified
|
|
425
|
+
|
|
426
|
+
Every claim here was produced by a run, and the command is quoted with it so you can check it
|
|
427
|
+
yourself.
|
|
428
|
+
|
|
429
|
+
| claim | evidence |
|
|
430
|
+
|---|---|
|
|
431
|
+
| the loop, policy and surfaces work | `pytest`: 147 passed, no credentials, no network |
|
|
432
|
+
| the browser surface drives a real browser | `pytest -m live`: launches headless Chrome against a local fixture page and reaches its postcondition |
|
|
433
|
+
| the package installs and runs as a package | `uv build`, then install the wheel into a fresh venv with no extras: `jev-pilot selftest` prints PASS |
|
|
434
|
+
| CI on three platforms | GitHub Actions on `main`: 3.9, 3.11 on Ubuntu, Windows and macOS, 3.13, and the live browser job, all green |
|
|
435
|
+
| a live site, real model, through the CLI | the transcript above: three hops, 1454 / 297 / 295 ms, $0.00086 |
|
|
436
|
+
| the same chain through the library API | `python examples/browser_chain.py --provider jev --headless`: 3 of 3 hops reached |
|
|
437
|
+
| a real OpenAI-compatible endpoint | `jev-pilot decide --provider openai --model <a chat model> --file examples/decide-request.json` against a local OpenAI-compatible relay: correct pick in 4357 ms |
|
|
438
|
+
| the desktop surface, clicking | `jev-pilot run --surface desktop --aumid Microsoft.WindowsCalculator_8wekyb3d8bbwe!App --provider jev --steps 6 --verify text-contains:8`: four real clicks, display verified from the window's own text, exit code 0, $0.000247 |
|
|
439
|
+
| the desktop path is covered without a window | `tests/test_desktop_loop.py` drives a scripted driver through a full episode: observe, pick, click by element token, verify from the window's own text |
|
|
440
|
+
| the frozen-state bench | the table above, `examples/bench-calculator.json`, three repeats per chooser |
|
|
441
|
+
| the same tasks against a second chooser | the table above: 6 of 6 tasks reached against 4 of 6, two passes each |
|
|
442
|
+
| the manual live lane runs on a GitHub runner | `live-jev.yml`, dispatched by hand with the API key as a repository secret: three hops, 277 / 124 / 127 ms, $0.00086 |
|
|
443
|
+
|
|
444
|
+
Still not verified, and said plainly rather than buried: a success rate over a task set larger
|
|
445
|
+
than three, and any comparison that would support a claim about judgment rather than reliability
|
|
446
|
+
(the note above explains why the loop comparison does not support one). The desktop surface has
|
|
447
|
+
had one real episode, in one application, on one platform; issue #2 is that gap.
|
|
448
|
+
|
|
449
|
+
## Requirements
|
|
450
|
+
|
|
451
|
+
- Python 3.9 to 3.13 (`[jev]` needs 3.10+).
|
|
452
|
+
- Chrome, Chromium or Edge for the browser surface. Headless is supported; set `JEV_PILOT_CHROME`
|
|
453
|
+
to point at a specific binary and `JEV_PILOT_CHROME_ARGS` for extra flags (containers and CI
|
|
454
|
+
usually need `--no-sandbox --disable-dev-shm-usage`).
|
|
455
|
+
- [cua-driver](https://github.com/trycua/cua) for the desktop surface, and a Windows session that
|
|
456
|
+
is actually on the interactive desktop.
|
|
457
|
+
- No credentials for the core, the browser surface, the mock chooser or the tests.
|
|
458
|
+
|
|
459
|
+
## Status
|
|
460
|
+
|
|
461
|
+
`building`, released as `0.1.0` (alpha). Started 2026-09-18. What changed is in `CHANGELOG.md`;
|
|
462
|
+
what has been verified, and what has not, is in `docs/findings.md`. Roadmap items live as issues
|
|
463
|
+
rather than in this file.
|
|
464
|
+
|
|
465
|
+
## Where things live
|
|
466
|
+
|
|
467
|
+
- `src/jev_pilot/`: the library (loop, policy, perception, safety, traces, bench, CLI, surfaces).
|
|
468
|
+
- `tests/`: 147 credential-free tests, plus one `live` test that drives a real headless Chrome.
|
|
469
|
+
- `docs/`: architecture, perception, decisions, cookbook, findings, parity with the wider ecosystem
|
|
470
|
+
effort, publishing, evidence for the quoted runs, and a decision log that records which calls were
|
|
471
|
+
made by a model and which by hand.
|
|
472
|
+
- `examples/`: a task file, a reference bench fixture, and runnable examples.
|
|
473
|
+
- `.github/workflows/`: CI (credential-free), the release lane, and an optional manual live lane.
|
|
474
|
+
|
|
475
|
+
## Contributing
|
|
476
|
+
|
|
477
|
+
See `CONTRIBUTING.md`. The short version: the default test suite must pass with no credentials and
|
|
478
|
+
no network, a bug fix comes with a test that fails before it, and a performance or accuracy claim
|
|
479
|
+
comes with the command that produced it.
|
|
480
|
+
|
|
481
|
+
## License
|
|
482
|
+
|
|
483
|
+
MIT. See `LICENSE`.
|