jev-browser-pilot 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. jev_browser_pilot-0.1.1/LICENSE +21 -0
  2. jev_browser_pilot-0.1.1/PKG-INFO +483 -0
  3. jev_browser_pilot-0.1.1/README.md +457 -0
  4. jev_browser_pilot-0.1.1/pyproject.toml +54 -0
  5. jev_browser_pilot-0.1.1/setup.cfg +4 -0
  6. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/PKG-INFO +483 -0
  7. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/SOURCES.txt +41 -0
  8. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/dependency_links.txt +1 -0
  9. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/entry_points.txt +2 -0
  10. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/requires.txt +10 -0
  11. jev_browser_pilot-0.1.1/src/jev_browser_pilot.egg-info/top_level.txt +1 -0
  12. jev_browser_pilot-0.1.1/src/jev_pilot/__init__.py +66 -0
  13. jev_browser_pilot-0.1.1/src/jev_pilot/bench.py +178 -0
  14. jev_browser_pilot-0.1.1/src/jev_pilot/browser.py +298 -0
  15. jev_browser_pilot-0.1.1/src/jev_pilot/cli.py +554 -0
  16. jev_browser_pilot-0.1.1/src/jev_pilot/desktop.py +234 -0
  17. jev_browser_pilot-0.1.1/src/jev_pilot/loop.py +214 -0
  18. jev_browser_pilot-0.1.1/src/jev_pilot/perception.py +275 -0
  19. jev_browser_pilot-0.1.1/src/jev_pilot/policy.py +100 -0
  20. jev_browser_pilot-0.1.1/src/jev_pilot/providers/__init__.py +48 -0
  21. jev_browser_pilot-0.1.1/src/jev_pilot/providers/base.py +147 -0
  22. jev_browser_pilot-0.1.1/src/jev_pilot/providers/jev.py +116 -0
  23. jev_browser_pilot-0.1.1/src/jev_pilot/providers/openai_compat.py +164 -0
  24. jev_browser_pilot-0.1.1/src/jev_pilot/safety.py +99 -0
  25. jev_browser_pilot-0.1.1/src/jev_pilot/serialize.py +91 -0
  26. jev_browser_pilot-0.1.1/src/jev_pilot/surface.py +71 -0
  27. jev_browser_pilot-0.1.1/src/jev_pilot/trace.py +203 -0
  28. jev_browser_pilot-0.1.1/src/jev_pilot/types.py +181 -0
  29. jev_browser_pilot-0.1.1/src/jev_pilot/verify.py +169 -0
  30. jev_browser_pilot-0.1.1/tests/test_bench.py +77 -0
  31. jev_browser_pilot-0.1.1/tests/test_browser.py +99 -0
  32. jev_browser_pilot-0.1.1/tests/test_cli.py +236 -0
  33. jev_browser_pilot-0.1.1/tests/test_desktop.py +63 -0
  34. jev_browser_pilot-0.1.1/tests/test_desktop_loop.py +146 -0
  35. jev_browser_pilot-0.1.1/tests/test_jev_provider.py +99 -0
  36. jev_browser_pilot-0.1.1/tests/test_loop.py +159 -0
  37. jev_browser_pilot-0.1.1/tests/test_perception.py +78 -0
  38. jev_browser_pilot-0.1.1/tests/test_policy.py +80 -0
  39. jev_browser_pilot-0.1.1/tests/test_providers.py +161 -0
  40. jev_browser_pilot-0.1.1/tests/test_safety.py +86 -0
  41. jev_browser_pilot-0.1.1/tests/test_serialize.py +72 -0
  42. jev_browser_pilot-0.1.1/tests/test_trace.py +75 -0
  43. jev_browser_pilot-0.1.1/tests/test_verify.py +66 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Aidil Zafri
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,483 @@
1
+ Metadata-Version: 2.4
2
+ Name: jev-browser-pilot
3
+ Version: 0.1.1
4
+ Summary: A bounded, cheap decision layer for browser and desktop automation: a System One model picks the next step, your code owns observation, content, actuation and verification.
5
+ Author-email: Aidil Zafri <aidilzafri56@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/aidil2105/jev-browser-pilot
8
+ Project-URL: Issues, https://github.com/aidil2105/jev-browser-pilot/issues
9
+ Keywords: computer-use,browser-automation,agents,decision-model,jev,typesafe,cdp
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Software Development :: Testing
15
+ Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
16
+ Requires-Python: >=3.9
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: websocket-client>=1.6
20
+ Provides-Extra: jev
21
+ Requires-Dist: typesafe-sdk>=0.6; python_version >= "3.10" and extra == "jev"
22
+ Provides-Extra: dev
23
+ Requires-Dist: pytest>=7; extra == "dev"
24
+ Requires-Dist: pytest-cov; extra == "dev"
25
+ Dynamic: license-file
26
+
27
+ # jev-browser-pilot
28
+
29
+ [![CI](https://github.com/aidil2105/jev-browser-pilot/actions/workflows/ci.yml/badge.svg)](https://github.com/aidil2105/jev-browser-pilot/actions/workflows/ci.yml)
30
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
31
+ [![Python 3.9 to 3.13](https://img.shields.io/badge/python-3.9%20to%203.13-blue.svg)](pyproject.toml)
32
+
33
+ A bounded decision layer for browser and desktop automation. Your code observes the surface,
34
+ builds the list of things it could do, executes one of them, and checks the result. A
35
+ decision-only model makes exactly one choice per step: which candidate is next.
36
+
37
+ ```
38
+ observe (your code) -> decide (the model, one typed choice) -> act (your code) -> verify (your code)
39
+ ```
40
+
41
+ The loop is not new. What is different is the split. The model never sees a screenshot, never
42
+ writes a selector, never produces text to type, and is never asked whether it succeeded. It
43
+ answers one bounded question over a list of ids your code built, and returns a confidence your
44
+ code routes on. Everything that can fail silently stays in code.
45
+
46
+ That buys two things: a step that costs a fraction of a cent and answers in a few hundred
47
+ milliseconds, and an audit trail that shows exactly what the model was shown at every step.
48
+
49
+ ## Install
50
+
51
+ Not on PyPI yet, so `pip install jev-browser-pilot` does not resolve. Use the repository:
52
+
53
+ ```
54
+ pip install "git+https://github.com/aidil2105/jev-browser-pilot"
55
+ # with the Jev provider (Python 3.10 or newer):
56
+ pip install "jev-browser-pilot[jev] @ git+https://github.com/aidil2105/jev-browser-pilot"
57
+ ```
58
+
59
+ Or from a clone, which is what the commands below assume:
60
+
61
+ ```
62
+ git clone https://github.com/aidil2105/jev-browser-pilot
63
+ cd jev-browser-pilot
64
+ python -m venv .venv
65
+ .venv/Scripts/python -m pip install -e ".[dev,jev]" # Windows
66
+ # .venv/bin/python -m pip install -e ".[dev,jev]" # macOS and Linux
67
+ ```
68
+
69
+ The only runtime dependency is `websocket-client`. The browser surface uses the Chrome,
70
+ Chromium or Edge you already have. The Jev provider needs Python 3.10 or newer because the
71
+ vendor SDK does; on 3.9 the core, the OpenAI-compatible provider and the mock chooser all work
72
+ and the Jev provider says so plainly instead of failing to install.
73
+
74
+ Releases are built and uploaded by `.github/workflows/publish.yml` when a GitHub release is
75
+ published, using PyPI's trusted publishing rather than a stored token; `docs/publishing.md` has
76
+ the one-time PyPI setup and the local fallback.
77
+
78
+ ## Quickstart
79
+
80
+ Three commands, in this order. The first one needs nothing at all.
81
+
82
+ **1. Prove the loop works, offline.** No browser, no network, no credentials:
83
+
84
+ ```
85
+ $ jev-pilot selftest
86
+ jev-browser-pilot 0.1.0 selftest
87
+ step 1: act pick=2 link 'Omega appendix' 0ms
88
+ pass: url-contains:omega.html
89
+ pass: text-contains:goal reached
90
+ reached=True stopped=reached steps=1 actions=['2']
91
+ pass: unknown id is an error
92
+ pass: low confidence escalates with the pick
93
+ pass: transport failure is an error
94
+ selftest: PASS
95
+ ```
96
+
97
+ **2. Borrow just the decision step.** JSON in, JSON out, so any harness can use it without
98
+ adopting the loop:
99
+
100
+ ```
101
+ $ echo '{"goal":"open the Omega appendix",
102
+ "state":"CANDIDATE ACTIONS:\n[1] link (Alpha report)\n[2] link (Omega appendix)",
103
+ "options":[{"id":"1","label":"link (Alpha report)"},
104
+ {"id":"2","label":"link (Omega appendix)"}]}' \
105
+ | jev-pilot decide --provider mock
106
+ {
107
+ "outcome": "act",
108
+ "pick": "2",
109
+ "label": "link (Omega appendix)",
110
+ "confidence": 0.8,
111
+ "provider": "keyword",
112
+ "reason": "picked"
113
+ }
114
+ ```
115
+
116
+ Output trimmed to the fields that matter; the real object also carries the full probability map,
117
+ tokens, cost and latency. The mock provider reports itself as `keyword`, which is what it is: a
118
+ deterministic keyword overlap, so the quickstart needs no key and no network.
119
+
120
+ **3. Drive a real site.** The browser gets its own profile and its own debugging port, and the
121
+ hosts it may touch are declared up front:
122
+
123
+ ```
124
+ $ jev-pilot run --start "https://en.wikipedia.org/wiki/Calculator" \
125
+ --task-file examples/wikipedia-chain.json \
126
+ --provider jev --headless --trace run.jsonl --report run.html
127
+ ```
128
+
129
+ Exit codes are part of the contract, because this is meant to run in a pipeline: `0` every goal
130
+ reached its postcondition, `1` a goal did not reach it, `2` a hard error (no browser, no
131
+ credentials, a safety refusal, bad input).
132
+
133
+ ## What a run looks like
134
+
135
+ Real output, three chained goals in one browser session, one decision per hop:
136
+
137
+ ```
138
+ jev-browser-pilot 0.1.0
139
+ surface: browser | chooser: jev model=jev-latest
140
+ safety: hosts=[en.wikipedia.org] actions=['click'] typing=off dry_run=False max_steps=12
141
+ session: chrome pid=9804 port=33154 profile=33154
142
+
143
+ === goal: open the article about the slide rule
144
+ step 1: act conf=0.95 pick=100 a 'slide rule' 1454ms
145
+ pass: url-contains:Slide_rule ('Slide_rule' in 'https://en.wikipedia.org/wiki/Slide_rule')
146
+ result: reached=True stopped=reached steps=1 median=1454ms cost=$0.000270
147
+
148
+ === goal: open the article about William Oughtred, the inventor of the slide rule
149
+ step 1: act conf=0.99 pick=14 a 'William Oughtred' 297ms
150
+ pass: url-contains:William_Oughtred ('William_Oughtred' in 'https://en.wikipedia.org/wiki/William_Oughtred')
151
+ result: reached=True stopped=reached steps=1 median=297ms cost=$0.000286
152
+
153
+ === goal: open the article about Richard Delamain, the rival who claimed priority for the slide rule
154
+ step 1: act conf=0.47 pick=88 a 'Richard Delamain' 295ms
155
+ pass: url-contains:Delamain ('Delamain' in 'https://en.wikipedia.org/wiki/Richard_Delamaine')
156
+ result: reached=True stopped=reached steps=1 median=295ms cost=$0.000301
157
+ report: run.html
158
+ ```
159
+
160
+ Three goals, three decisions, $0.00086 for the session. Note the third hop: it is the only one
161
+ whose goal describes a person rather than nearly quoting the link text, and it is the only one
162
+ answered at 0.47 confidence. Right answer, visibly less sure. That confidence is the signal you
163
+ route on.
164
+ ## Why this shape
165
+
166
+ An agent that reads a page with a full language model pays seconds and cents per step for a
167
+ judgment that is often a single pick among a handful of visible options. Measured on five frozen
168
+ observations of a real window, three repeats each, no capture and no actuation involved, so this is
169
+ the decision alone (`examples/bench-calculator.json`):
170
+
171
+ | chooser | correct | accuracy | answered | median | range | cost |
172
+ |---|---|---|---|---|---|---|
173
+ | `jev` | 15/15 | 100% | 15/15 | 349 ms | 277 to 870 ms | $0.000393 |
174
+ | `openai`, a mid-tier chat model | 12/15 | 80% | 12/15 | 2,858 ms | 4 to 15,328 ms | not metered here |
175
+ | `mock`, keyword overlap | 3/15 | 20% | 15/15 | 0 ms | 0 ms | free |
176
+
177
+ Read that honestly: the chat model's three misses are all the same transport failure on the hardest
178
+ state (a relay 503, not a wrong answer), so the column that matters most between it and `jev` here
179
+ is latency. `jev` answered every call, never wrong, at 0.93 to 1.00 confidence on the four
180
+ deterministic states and 0.78 on the one that relies on reading a display. The `mock` chooser fails
181
+ every state whose answer is not in the goal's own words, which is exactly what a naive keyword
182
+ overlap should do.
183
+
184
+ Reproduce it, or run it against your own states:
185
+
186
+ ```
187
+ jev-pilot bench --fixtures examples/bench-calculator.json \
188
+ --chooser jev --chooser mock --chooser openai:llama3.1 \
189
+ --base-url http://localhost:11434/v1 --repeats 3
190
+ ```
191
+
192
+ The `mock` chooser is a deterministic keyword overlap stand-in, deliberately naive: it exists so
193
+ the library, its tests and its docs work with no key, no network and no spend. It is a fixture,
194
+ not a baseline to beat.
195
+
196
+ ## What this is not
197
+
198
+ - **Not an autonomous agent.** There is no lookahead. If the next step is not derivable from the
199
+ current observation, the model answers `__stuck__` instead of guessing. Put a planner above it.
200
+ - **Not a text generator.** It cannot produce the string to type, the URL to open or the value to
201
+ set. Your code owns content. It also means no prompt injection can make it invent one.
202
+ - **Not a success oracle.** Postconditions are evaluated by code, before and after every action.
203
+ The model's `done` is a report, not a proof.
204
+
205
+ ## How it fits together
206
+
207
+ | piece | what it does |
208
+ |---|---|
209
+ | `Surface` | whatever you are driving: `BrowserPilot`, `DesktopPilot`, `ScriptedSurface`, or your own object with `observe`/`act`/`url`/`probe` |
210
+ | `perception` | turns a page or a window into `Element`s and mints the candidate `Option` ids |
211
+ | `serialize` | renders the exact state string the model sees |
212
+ | `Chooser` | the decision backend: `jev`, `openai` (any compatible endpoint), `mock`, `scripted`, or yours |
213
+ | `policy` | what an answer is allowed to mean, in code rather than in a prompt |
214
+ | `verify` | deterministic postconditions: URL, text, CSS selector, file contents |
215
+ | `safety` | declared hosts, allowed actions, typing off by default, dry run |
216
+ | `trace` | JSONL per step plus a self-contained HTML report |
217
+ | `bench` | the same frozen states replayed against several choosers |
218
+
219
+ The library API is small enough to quote:
220
+
221
+ ```python
222
+ from jev_pilot import SafetyPolicy, run_episode
223
+ from jev_pilot.browser import BrowserPilot
224
+ from jev_pilot.providers import build_chooser
225
+
226
+ safety = SafetyPolicy.for_url("https://en.wikipedia.org/wiki/Calculator")
227
+ chooser = build_chooser("jev") # or "openai" with model=..., or "mock"
228
+
229
+ with BrowserPilot("https://en.wikipedia.org/wiki/Calculator", headless=True,
230
+ safety=safety) as pilot:
231
+ episode = run_episode(
232
+ pilot, chooser,
233
+ goal="open the article about the slide rule",
234
+ verifiers=["url-contains:Slide_rule"],
235
+ steps=3, floor=0.6, safety=safety,
236
+ )
237
+
238
+ print(episode.reached, episode.stopped, episode.summary())
239
+ ```
240
+
241
+ ## The decision policy
242
+
243
+ One question per step: a single choice over the candidate ids, plus two sentinels. The rules that
244
+ turn an answer into an outcome live in `policy.py`:
245
+
246
+ | outcome | meaning | who handles it |
247
+ |---|---|---|
248
+ | `act` | a candidate was chosen and is safe to execute | the loop clicks it |
249
+ | `done` | the model reports the goal is achieved | the loop stops; the postcondition still decides `reached` |
250
+ | `stuck` | nothing in the list can make progress | the loop stops; a planner above decides what next |
251
+ | `escalate` | the pick is under the confidence floor, **with the pick attached** | your planner, or a human |
252
+ | `error` | unknown id, empty answer, transport failure | your caller; never treated as a pick |
253
+
254
+ Where an episode can stop is equally explicit: `reached`, `done`, `stuck`, `escalate`, `error`,
255
+ `blocked` (the surface left the declared hosts), `no_progress` (the same state twice in a row),
256
+ `budget`, `dry_run`.
257
+
258
+ Asking one question instead of two is deliberate: an earlier version asked for the action and the
259
+ target separately and produced contradictory answers, picking an element while also declaring the
260
+ task done. One question, one answer, one decision.
261
+
262
+ ## Providers, including yours
263
+
264
+ | name | what it is | notes |
265
+ |---|---|---|
266
+ | `jev` | TypeSafe Jev, a decision-only model | needs `TYPESAFE_API_KEY`; returns the full probability map |
267
+ | `openai` | any OpenAI-compatible endpoint | Ollama, vLLM, LM Studio, a gateway; needs `--model`, no confidence returned |
268
+ | `mock` | deterministic keyword overlap | credential-free, for tests, docs and offline development |
269
+ | `scripted` | a fixed sequence of answers | assertions in tests |
270
+
271
+ A chooser is one method, so adding yours does not touch the loop, the safety rails or the
272
+ verification:
273
+
274
+ ```python
275
+ from jev_pilot.providers.base import Chooser, RawAnswer
276
+
277
+ class MyChooser(Chooser):
278
+ name = "mine"
279
+
280
+ def choose(self, *, goal, state, options, timeout=None):
281
+ answer = my_service(goal, state, [o.as_dict() for o in options])
282
+ return RawAnswer(pick=answer["id"], confidence=answer.get("confidence"))
283
+ ```
284
+
285
+ `RawAnswer` is not permission to act. It goes through `apply_policy`, which is where the floor,
286
+ the sentinels and the fail-closed rules apply to every provider equally.
287
+
288
+ ## Postconditions
289
+
290
+ Verification is a list of specs, checked before the first decision and after every action. The
291
+ success decision never involves the model.
292
+
293
+ | spec | passes when |
294
+ |---|---|
295
+ | `url-contains:TEXT` | the current URL contains TEXT (case sensitive) |
296
+ | `url-regex:PATTERN` | the URL matches the pattern |
297
+ | `text-contains:TEXT` | the visible text contains TEXT (case and whitespace insensitive) |
298
+ | `selector:CSS` | the page has at least one matching element |
299
+ | `file-contains:PATH::TEXT` | the file exists and contains TEXT |
300
+
301
+ Prefer a postcondition the page cannot fake. `text-contains:` is the weakest of them, because
302
+ marketing copy that merely mentions the word passes it. A minted order id, a confirmation element
303
+ or a file that a download had to produce are all stronger.
304
+ ## Safety rails
305
+
306
+ Opt-in by construction, because this drives a real browser on a real machine.
307
+
308
+ - **Hosts are declared, not inferred.** Navigation is confined to the hosts you name (a `file://`
309
+ page reports the pseudo-host `file`). The moment an observation lands somewhere else, the
310
+ episode stops with `blocked`, that outranks a postcondition which would have passed there, and a
311
+ chain abandons its remaining tasks.
312
+ - **Only `click` is allowed** by default. Typing is off until you enable it, and there is no API in
313
+ this library for passwords, tokens or card numbers at all.
314
+ - **The browser is isolated.** Its own `--user-data-dir` and its own debugging port, so it cannot
315
+ attach to a logged-in profile.
316
+ - **`--dry-run` decides and records, and stops before anything is actuated.** Useful when wiring a
317
+ new surface, and useful for showing what the model would have done.
318
+
319
+ The rails are checked by the loop, not requested of the model, so a provider that hallucinates a
320
+ better plan cannot walk around them.
321
+
322
+ ## Benchmarks over frozen states
323
+
324
+ A live comparison mostly measures who got luckier with perception. `bench` removes that variable:
325
+ one observation, frozen to a file, replayed against several choosers, scored against a known
326
+ answer.
327
+
328
+ ```json
329
+ {"cases": [
330
+ {"name": "after-plus",
331
+ "category": "calculator",
332
+ "goal": "compute 5 + 3, stopping when the display shows the answer",
333
+ "state": "DISPLAY: 'Display is 5'\nACTIONS ALREADY TAKEN THIS EPISODE: Five, Plus\n[23] Button 'Plus'\n[24] Button 'Equals'\n[28] Button 'Three'",
334
+ "options": [{"id": "23", "label": "Button 'Plus'"},
335
+ {"id": "24", "label": "Button 'Equals'"},
336
+ {"id": "28", "label": "Button 'Three'"}],
337
+ "expect": "Button 'Three'"}
338
+ ]}
339
+ ```
340
+
341
+ `expect` is a label, an `id:`, or a sentinel (`__done__`, `__stuck__`). The report gives you
342
+ accuracy next to **coverage**, because a chooser that answers 6 of 10 calls and gets them right is
343
+ not the same as one that answers all 10.
344
+
345
+ ```
346
+ jev-pilot bench --fixtures examples/bench-calculator.json --chooser jev --repeats 3 \
347
+ --markdown bench.md --out bench.json
348
+ ```
349
+
350
+ ### The same tasks, two choosers, twice each
351
+
352
+ The frozen states above isolate a single decision. This runs the whole loop instead: one task
353
+ file, one start page, the same postconditions, one arm per chooser, two passes, headless Chrome.
354
+
355
+ ```
356
+ jev-pilot run --start https://en.wikipedia.org/wiki/Calculator \
357
+ --task-file examples/wikipedia-chain.json --provider jev --headless
358
+
359
+ jev-pilot run --start https://en.wikipedia.org/wiki/Calculator \
360
+ --task-file examples/wikipedia-chain.json --provider openai \
361
+ --model <a free open-weights chat model> --base-url <an OpenAI-compatible relay> --headless
362
+ ```
363
+
364
+ | chooser | tasks reached | per-decision latency | per-decision cost |
365
+ |---|---|---|---|
366
+ | `jev` | 6 of 6 | 302 to 982 ms | $0.00027 to $0.00030 |
367
+ | a free open-weights chat model | 4 of 6 | 2,733 to 51,665 ms | unmetered, free route |
368
+
369
+ The four run summaries behind this table are committed under `docs/evidence/live-comparison/`, so
370
+ the numbers can be read back instead of taken on trust.
371
+
372
+ What this does and does not say. Whenever the chat model answered it picked the same element Jev
373
+ picked, every time, so the two agreed on all four decisions it completed. Its two failures were
374
+ not wrong picks: they were calls that never returned, both on the same task, at 30 s and 52 s,
375
+ from a relay that answered with an empty response where Jev answered in 302 ms at 0.49
376
+ confidence. Read it as a reliability and latency result under this harness, not as a ranking of
377
+ two models' judgment. Three tasks, one site, two passes: enough to show the shape of the
378
+ difference, not enough to generalise past it.
379
+
380
+ It also shows the exit codes doing their job: the arm that missed a task exited 1, the arm that
381
+ reached all three exited 0.
382
+
383
+ ## Exit codes
384
+
385
+ Running `run` from a script is the point, so the codes are part of the interface.
386
+
387
+ | code | meaning |
388
+ |---|---|
389
+ | 0 | every task reached its postcondition |
390
+ | 1 | at least one task was not reached: the postcondition failed, the chooser never answered, or the episode stopped `blocked` because the surface left the declared hosts |
391
+ | 2 | a usage or configuration problem: an unknown chooser, an unreadable fixture, a missing credential |
392
+
393
+ Confidence never turns a miss into a success. An episode where every decision was confident and
394
+ the postcondition still failed exits 1, and the run prints which task it was.
395
+
396
+ ## Traces and reports
397
+
398
+ Every step is recorded: the exact state sent, the ids offered, the answer, the confidence, the
399
+ latency, the tokens and the cost. `jev-pilot report` renders it into a single self-contained HTML
400
+ file with no external assets, so it can be attached to a bug report or a pull request.
401
+
402
+ ```
403
+ jev-pilot run ... --trace run.jsonl --report run.html
404
+ jev-pilot report --trace run.jsonl --out run.html # render later
405
+ ```
406
+
407
+ Use `--no-verbatim-state` when the page content must not be persisted. The decision is still
408
+ recorded; only the state text is left out.
409
+
410
+ ## Known limits
411
+
412
+ - **Perception is the ceiling.** On a Wikipedia article, 200 of 447 candidate links fit the cap.
413
+ What falls past it is invisible, so a correct `__stuck__` can mean "I cannot see it" rather than
414
+ "it is not there". Check `matched_total` in the trace before blaming the model.
415
+ - **Confidence is a router, not a gate you set once.** The same ambiguous hop scored 0.47 in one
416
+ run, 0.49 in another and 0.61 in two more. Set the floor from the distribution you observe on
417
+ your own task.
418
+ - **No lookahead, no typing, no re-planning.** Those belong to the layer above.
419
+ - **The desktop surface is experimental.** It drives a native window through a UI Automation tree
420
+ via cua-driver. It has completed a real four-click episode (see below), but UIA quality varies
421
+ by application and it has had far less use than the browser surface.
422
+ - **No screenshot perception.** Text states only, by design: cheaper, auditable and diffable.
423
+
424
+ ## What has been verified
425
+
426
+ Every claim here was produced by a run, and the command is quoted with it so you can check it
427
+ yourself.
428
+
429
+ | claim | evidence |
430
+ |---|---|
431
+ | the loop, policy and surfaces work | `pytest`: 147 passed, no credentials, no network |
432
+ | the browser surface drives a real browser | `pytest -m live`: launches headless Chrome against a local fixture page and reaches its postcondition |
433
+ | the package installs and runs as a package | `uv build`, then install the wheel into a fresh venv with no extras: `jev-pilot selftest` prints PASS |
434
+ | CI on three platforms | GitHub Actions on `main`: 3.9, 3.11 on Ubuntu, Windows and macOS, 3.13, and the live browser job, all green |
435
+ | a live site, real model, through the CLI | the transcript above: three hops, 1454 / 297 / 295 ms, $0.00086 |
436
+ | the same chain through the library API | `python examples/browser_chain.py --provider jev --headless`: 3 of 3 hops reached |
437
+ | a real OpenAI-compatible endpoint | `jev-pilot decide --provider openai --model <a chat model> --file examples/decide-request.json` against a local OpenAI-compatible relay: correct pick in 4357 ms |
438
+ | the desktop surface, clicking | `jev-pilot run --surface desktop --aumid Microsoft.WindowsCalculator_8wekyb3d8bbwe!App --provider jev --steps 6 --verify text-contains:8`: four real clicks, display verified from the window's own text, exit code 0, $0.000247 |
439
+ | the desktop path is covered without a window | `tests/test_desktop_loop.py` drives a scripted driver through a full episode: observe, pick, click by element token, verify from the window's own text |
440
+ | the frozen-state bench | the table above, `examples/bench-calculator.json`, three repeats per chooser |
441
+ | the same tasks against a second chooser | the table above: 6 of 6 tasks reached against 4 of 6, two passes each |
442
+ | the manual live lane runs on a GitHub runner | `live-jev.yml`, dispatched by hand with the API key as a repository secret: three hops, 277 / 124 / 127 ms, $0.00086 |
443
+
444
+ Still not verified, and said plainly rather than buried: a success rate over a task set larger
445
+ than three, and any comparison that would support a claim about judgment rather than reliability
446
+ (the note above explains why the loop comparison does not support one). The desktop surface has
447
+ had one real episode, in one application, on one platform; issue #2 is that gap.
448
+
449
+ ## Requirements
450
+
451
+ - Python 3.9 to 3.13 (`[jev]` needs 3.10+).
452
+ - Chrome, Chromium or Edge for the browser surface. Headless is supported; set `JEV_PILOT_CHROME`
453
+ to point at a specific binary and `JEV_PILOT_CHROME_ARGS` for extra flags (containers and CI
454
+ usually need `--no-sandbox --disable-dev-shm-usage`).
455
+ - [cua-driver](https://github.com/trycua/cua) for the desktop surface, and a Windows session that
456
+ is actually on the interactive desktop.
457
+ - No credentials for the core, the browser surface, the mock chooser or the tests.
458
+
459
+ ## Status
460
+
461
+ `building`, released as `0.1.0` (alpha). Started 2026-09-18. What changed is in `CHANGELOG.md`;
462
+ what has been verified, and what has not, is in `docs/findings.md`. Roadmap items live as issues
463
+ rather than in this file.
464
+
465
+ ## Where things live
466
+
467
+ - `src/jev_pilot/`: the library (loop, policy, perception, safety, traces, bench, CLI, surfaces).
468
+ - `tests/`: 147 credential-free tests, plus one `live` test that drives a real headless Chrome.
469
+ - `docs/`: architecture, perception, decisions, cookbook, findings, parity with the wider ecosystem
470
+ effort, publishing, evidence for the quoted runs, and a decision log that records which calls were
471
+ made by a model and which by hand.
472
+ - `examples/`: a task file, a reference bench fixture, and runnable examples.
473
+ - `.github/workflows/`: CI (credential-free), the release lane, and an optional manual live lane.
474
+
475
+ ## Contributing
476
+
477
+ See `CONTRIBUTING.md`. The short version: the default test suite must pass with no credentials and
478
+ no network, a bug fix comes with a test that fails before it, and a performance or accuracy claim
479
+ comes with the command that produced it.
480
+
481
+ ## License
482
+
483
+ MIT. See `LICENSE`.