web-cortex-framework 2.1.1__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/Cargo.lock +48 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/Cargo.toml +4 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/PKG-INFO +84 -4
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/README.md +83 -3
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/Cargo.toml +1 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/provider.rs +70 -14
- web_cortex_framework-2.3.0/crates/webcortex-core/src/agent/vision.rs +348 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent.rs +249 -14
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/app.rs +178 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/auth.rs +5 -0
- web_cortex_framework-2.3.0/crates/webcortex-core/src/devices.rs +718 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/lib.rs +2 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/manifest.rs +196 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/mcp.rs +28 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/server.rs +60 -2
- web_cortex_framework-2.3.0/crates/webcortex-core/src/ws.rs +341 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/src/behaviour.rs +2 -4
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/pyproject.toml +1 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/__init__.py +3 -1
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/_bridge.py +4 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/app.py +266 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/cli.py +64 -1
- web_cortex_framework-2.3.0/python/webcortex/client.py +292 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/contextpack.py +32 -3
- web_cortex_framework-2.3.0/python/webcortex/media.py +240 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/starters.py +295 -2
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/LICENSE +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/registry.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/session.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/audit.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/bridge.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/context.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/db.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/files.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/flow.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/http.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/ledger.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/middleware.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/openapi.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/router.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/templates.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/typegen.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/Cargo.toml +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/src/lib.rs +0 -0
- {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/schema.py +0 -0
|
@@ -286,6 +286,12 @@ dependencies = [
|
|
|
286
286
|
"syn 2.0.119",
|
|
287
287
|
]
|
|
288
288
|
|
|
289
|
+
[[package]]
|
|
290
|
+
name = "data-encoding"
|
|
291
|
+
version = "2.11.1"
|
|
292
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
293
|
+
checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
|
294
|
+
|
|
289
295
|
[[package]]
|
|
290
296
|
name = "der"
|
|
291
297
|
version = "0.7.10"
|
|
@@ -1824,6 +1830,17 @@ dependencies = [
|
|
|
1824
1830
|
"serde",
|
|
1825
1831
|
]
|
|
1826
1832
|
|
|
1833
|
+
[[package]]
|
|
1834
|
+
name = "sha1"
|
|
1835
|
+
version = "0.11.0"
|
|
1836
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1837
|
+
checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
|
|
1838
|
+
dependencies = [
|
|
1839
|
+
"cfg-if",
|
|
1840
|
+
"cpufeatures 0.3.0",
|
|
1841
|
+
"digest 0.11.3",
|
|
1842
|
+
]
|
|
1843
|
+
|
|
1827
1844
|
[[package]]
|
|
1828
1845
|
name = "sha2"
|
|
1829
1846
|
version = "0.10.9"
|
|
@@ -2304,6 +2321,18 @@ dependencies = [
|
|
|
2304
2321
|
"tokio",
|
|
2305
2322
|
]
|
|
2306
2323
|
|
|
2324
|
+
[[package]]
|
|
2325
|
+
name = "tokio-tungstenite"
|
|
2326
|
+
version = "0.30.0"
|
|
2327
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2328
|
+
checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71"
|
|
2329
|
+
dependencies = [
|
|
2330
|
+
"futures-util",
|
|
2331
|
+
"log",
|
|
2332
|
+
"tokio",
|
|
2333
|
+
"tungstenite",
|
|
2334
|
+
]
|
|
2335
|
+
|
|
2307
2336
|
[[package]]
|
|
2308
2337
|
name = "tokio-util"
|
|
2309
2338
|
version = "0.7.19"
|
|
@@ -2431,6 +2460,22 @@ version = "0.2.5"
|
|
|
2431
2460
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2432
2461
|
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
|
2433
2462
|
|
|
2463
|
+
[[package]]
|
|
2464
|
+
name = "tungstenite"
|
|
2465
|
+
version = "0.30.0"
|
|
2466
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2467
|
+
checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
|
|
2468
|
+
dependencies = [
|
|
2469
|
+
"bytes",
|
|
2470
|
+
"data-encoding",
|
|
2471
|
+
"http",
|
|
2472
|
+
"httparse",
|
|
2473
|
+
"log",
|
|
2474
|
+
"rand 0.10.2",
|
|
2475
|
+
"sha1",
|
|
2476
|
+
"thiserror",
|
|
2477
|
+
]
|
|
2478
|
+
|
|
2434
2479
|
[[package]]
|
|
2435
2480
|
name = "typenum"
|
|
2436
2481
|
version = "1.20.1"
|
|
@@ -2616,7 +2661,7 @@ dependencies = [
|
|
|
2616
2661
|
|
|
2617
2662
|
[[package]]
|
|
2618
2663
|
name = "webcortex-core"
|
|
2619
|
-
version = "2.
|
|
2664
|
+
version = "2.3.0"
|
|
2620
2665
|
dependencies = [
|
|
2621
2666
|
"base64 0.23.1",
|
|
2622
2667
|
"bytes",
|
|
@@ -2636,6 +2681,7 @@ dependencies = [
|
|
|
2636
2681
|
"sqlx",
|
|
2637
2682
|
"subtle",
|
|
2638
2683
|
"tokio",
|
|
2684
|
+
"tokio-tungstenite",
|
|
2639
2685
|
"tracing",
|
|
2640
2686
|
"tracing-subscriber",
|
|
2641
2687
|
"uuid",
|
|
@@ -2643,7 +2689,7 @@ dependencies = [
|
|
|
2643
2689
|
|
|
2644
2690
|
[[package]]
|
|
2645
2691
|
name = "webcortex-py"
|
|
2646
|
-
version = "2.
|
|
2692
|
+
version = "2.3.0"
|
|
2647
2693
|
dependencies = [
|
|
2648
2694
|
"bytes",
|
|
2649
2695
|
"futures",
|
|
@@ -3,7 +3,7 @@ members = ["crates/webcortex-core", "crates/webcortex-py"]
|
|
|
3
3
|
resolver = "3"
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "2.
|
|
6
|
+
version = "2.3.0"
|
|
7
7
|
edition = "2024"
|
|
8
8
|
license = "Apache-2.0"
|
|
9
9
|
rust-version = "1.85"
|
|
@@ -39,6 +39,9 @@ subtle = "2"
|
|
|
39
39
|
base64 = "0.23"
|
|
40
40
|
uuid = { version = "1", features = ["v4", "fast-rng"] }
|
|
41
41
|
mime_guess = "2"
|
|
42
|
+
# WebSockets: devices stream frames in, subscribers and agents stream out. Only the
|
|
43
|
+
# server handshake is used; the HTTP upgrade itself is hyper's.
|
|
44
|
+
tokio-tungstenite = { version = "0.30", default-features = false, features = ["handshake"] }
|
|
42
45
|
|
|
43
46
|
[profile.release]
|
|
44
47
|
opt-level = 3
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: web-cortex-framework
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
@@ -349,6 +349,84 @@ invoke.
|
|
|
349
349
|
**A full audit trail**, including refused calls, handoffs, compactions and
|
|
350
350
|
approval decisions, at `GET /_webcortex/audit`.
|
|
351
351
|
|
|
352
|
+
## Agents that see, machines that move
|
|
353
|
+
|
|
354
|
+
WebCortex embeds no vision model — it carries pixels to the model that has one,
|
|
355
|
+
and puts the guard rails a physical system needs around the tools that move it.
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
from webcortex import HTTPError, Image, WebCortex
|
|
359
|
+
|
|
360
|
+
app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
|
|
361
|
+
|
|
362
|
+
@app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
|
|
363
|
+
def camera() -> dict:
|
|
364
|
+
frame = cam.read() # an OpenCV / NumPy frame
|
|
365
|
+
return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
|
|
366
|
+
|
|
367
|
+
@app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
|
|
368
|
+
actuator=True, approval="required")
|
|
369
|
+
def move(joint: str, degrees: float) -> dict:
|
|
370
|
+
if not -90 <= degrees <= 90:
|
|
371
|
+
raise HTTPError(422, "out of range") # limits live in code, not prompts
|
|
372
|
+
return arm.move(joint, degrees)
|
|
373
|
+
|
|
374
|
+
app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
- **The agent sees the frame.** A tool that returns a `webcortex.Image` —
|
|
378
|
+
anywhere in its result — hands the model the picture itself, over both wire
|
|
379
|
+
formats (Anthropic and OpenAI-compatible, so local vision models too) and over
|
|
380
|
+
MCP, where Claude Code and other clients receive it as image content. Agent
|
|
381
|
+
endpoints also take `"images": [...]` with the input.
|
|
382
|
+
- **Frames do not become a token leak.** Images are lifted out before
|
|
383
|
+
tool-result bounding, step records keep their size rather than their base64,
|
|
384
|
+
and only the last `max_images` stay in context as pixels.
|
|
385
|
+
- **An emergency stop that cannot be routed around.** `actuator=True` routes
|
|
386
|
+
refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
|
|
387
|
+
flow, MCP, or an approval granted after the stop. `webcortex halt` /
|
|
388
|
+
`webcortex release`, or `POST /_webcortex/halt` and `/release`.
|
|
389
|
+
- **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
|
|
390
|
+
an inspector agent and an operator agent, gated, scoped and booting halted.
|
|
391
|
+
Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
|
|
392
|
+
|
|
393
|
+
See [Vision and robotics](docs/vision-and-robotics.md).
|
|
394
|
+
|
|
395
|
+
## The device hub: cameras in, insights out
|
|
396
|
+
|
|
397
|
+
WebCortex sits between devices and agents, and the Rust core does the work.
|
|
398
|
+
Cameras and sensors stream in, agents see what they send, and what the agents
|
|
399
|
+
conclude streams out to anyone subscribed.
|
|
400
|
+
|
|
401
|
+
```python
|
|
402
|
+
app.camera("dock", max_fps=10) # pushed: browser, Pi, ESP32, curl
|
|
403
|
+
app.camera("lobby", source="http://192.168.1.40/snapshot.jpg") # pulled: any IP camera
|
|
404
|
+
app.sensor("dock_env") # telemetry only
|
|
405
|
+
|
|
406
|
+
app.agent("inspector", tools=["dock_snapshot", "dock_env_telemetry"], max_images=2)
|
|
407
|
+
app.watch("dock_watch", device="dock", agent="inspector", every=15,
|
|
408
|
+
input="Anything unsafe at the dock?", max_runs_per_hour=120,
|
|
409
|
+
webhook="https://ops.example.com/hooks/dock")
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
- **Connect anything.** WebSocket (`/devices/dock/ws`: binary messages are
|
|
413
|
+
frames, text is telemetry), HTTP POST, or an IP camera's snapshot URL. Open
|
|
414
|
+
`/devices/dock/connect` on a phone or laptop and its camera becomes the
|
|
415
|
+
device.
|
|
416
|
+
- **No Python per frame.** Ring buffers, sniffing, per-device rate limits and
|
|
417
|
+
broadcast fan-out all live in Rust.
|
|
418
|
+
- **Agents see it.** `dock_snapshot` is a tool that returns the live frame as
|
|
419
|
+
an image the model sees, over MCP too.
|
|
420
|
+
- **Watchers are the middle of the hub.** When the picture changes, the agent
|
|
421
|
+
looks, and its insight goes to every `/devices/dock/stream` subscriber, to a
|
|
422
|
+
webhook, and to the audit log. Spend is capped per hour.
|
|
423
|
+
- **Agents over WebSocket.** Any agent route accepts a socket: send a turn,
|
|
424
|
+
receive each step live, then the result.
|
|
425
|
+
- **A stdlib Python client** (`webcortex.client`): `DeviceConnection` for
|
|
426
|
+
drivers, `subscribe()` for consumers, `AgentSocket` for services.
|
|
427
|
+
|
|
428
|
+
`webcortex new site -t hub` scaffolds it. See [The device hub](docs/device-hub.md).
|
|
429
|
+
|
|
352
430
|
## Security defaults
|
|
353
431
|
|
|
354
432
|
Deny-by-default throughout; relaxing something costs a line, tightening it costs
|
|
@@ -432,7 +510,7 @@ tested, the known limits, and what v2 added to the surface.
|
|
|
432
510
|
|
|
433
511
|
## Status
|
|
434
512
|
|
|
435
|
-
v2.
|
|
513
|
+
v2.3.0. Working and tested: the manifest IR, router, native ops (static /
|
|
436
514
|
query / proxy / page / files / flow), the free-threaded Python bridge,
|
|
437
515
|
authentication and scopes, rate limiting, CORS, security headers, graceful
|
|
438
516
|
shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
|
|
@@ -440,8 +518,10 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
|
|
|
440
518
|
compaction, context providers, memory, flows, two providers (Anthropic and
|
|
441
519
|
OpenAI-compatible, which covers local models), prompt caching, the spend
|
|
442
520
|
ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
|
|
443
|
-
context pack and `evolve
|
|
444
|
-
|
|
521
|
+
context pack and `evolve`, images in and out of agent runs and MCP, actuators
|
|
522
|
+
and the emergency stop, the device hub (cameras, sensors, watchers, WebSocket
|
|
523
|
+
ingest and streams, agents over WebSocket). **427 tests** (132 Rust, 295 Python, including a
|
|
524
|
+
54-test adversarial suite and offline end-to-end suites that drive the whole
|
|
445
525
|
agent stack over HTTP), clippy clean.
|
|
446
526
|
|
|
447
527
|
Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
|
|
@@ -321,6 +321,84 @@ invoke.
|
|
|
321
321
|
**A full audit trail**, including refused calls, handoffs, compactions and
|
|
322
322
|
approval decisions, at `GET /_webcortex/audit`.
|
|
323
323
|
|
|
324
|
+
## Agents that see, machines that move
|
|
325
|
+
|
|
326
|
+
WebCortex embeds no vision model — it carries pixels to the model that has one,
|
|
327
|
+
and puts the guard rails a physical system needs around the tools that move it.
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
from webcortex import HTTPError, Image, WebCortex
|
|
331
|
+
|
|
332
|
+
app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
|
|
333
|
+
|
|
334
|
+
@app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
|
|
335
|
+
def camera() -> dict:
|
|
336
|
+
frame = cam.read() # an OpenCV / NumPy frame
|
|
337
|
+
return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
|
|
338
|
+
|
|
339
|
+
@app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
|
|
340
|
+
actuator=True, approval="required")
|
|
341
|
+
def move(joint: str, degrees: float) -> dict:
|
|
342
|
+
if not -90 <= degrees <= 90:
|
|
343
|
+
raise HTTPError(422, "out of range") # limits live in code, not prompts
|
|
344
|
+
return arm.move(joint, degrees)
|
|
345
|
+
|
|
346
|
+
app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
- **The agent sees the frame.** A tool that returns a `webcortex.Image` —
|
|
350
|
+
anywhere in its result — hands the model the picture itself, over both wire
|
|
351
|
+
formats (Anthropic and OpenAI-compatible, so local vision models too) and over
|
|
352
|
+
MCP, where Claude Code and other clients receive it as image content. Agent
|
|
353
|
+
endpoints also take `"images": [...]` with the input.
|
|
354
|
+
- **Frames do not become a token leak.** Images are lifted out before
|
|
355
|
+
tool-result bounding, step records keep their size rather than their base64,
|
|
356
|
+
and only the last `max_images` stay in context as pixels.
|
|
357
|
+
- **An emergency stop that cannot be routed around.** `actuator=True` routes
|
|
358
|
+
refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
|
|
359
|
+
flow, MCP, or an approval granted after the stop. `webcortex halt` /
|
|
360
|
+
`webcortex release`, or `POST /_webcortex/halt` and `/release`.
|
|
361
|
+
- **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
|
|
362
|
+
an inspector agent and an operator agent, gated, scoped and booting halted.
|
|
363
|
+
Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
|
|
364
|
+
|
|
365
|
+
See [Vision and robotics](docs/vision-and-robotics.md).
|
|
366
|
+
|
|
367
|
+
## The device hub: cameras in, insights out
|
|
368
|
+
|
|
369
|
+
WebCortex sits between devices and agents, and the Rust core does the work.
|
|
370
|
+
Cameras and sensors stream in, agents see what they send, and what the agents
|
|
371
|
+
conclude streams out to anyone subscribed.
|
|
372
|
+
|
|
373
|
+
```python
|
|
374
|
+
app.camera("dock", max_fps=10) # pushed: browser, Pi, ESP32, curl
|
|
375
|
+
app.camera("lobby", source="http://192.168.1.40/snapshot.jpg") # pulled: any IP camera
|
|
376
|
+
app.sensor("dock_env") # telemetry only
|
|
377
|
+
|
|
378
|
+
app.agent("inspector", tools=["dock_snapshot", "dock_env_telemetry"], max_images=2)
|
|
379
|
+
app.watch("dock_watch", device="dock", agent="inspector", every=15,
|
|
380
|
+
input="Anything unsafe at the dock?", max_runs_per_hour=120,
|
|
381
|
+
webhook="https://ops.example.com/hooks/dock")
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
- **Connect anything.** WebSocket (`/devices/dock/ws`: binary messages are
|
|
385
|
+
frames, text is telemetry), HTTP POST, or an IP camera's snapshot URL. Open
|
|
386
|
+
`/devices/dock/connect` on a phone or laptop and its camera becomes the
|
|
387
|
+
device.
|
|
388
|
+
- **No Python per frame.** Ring buffers, sniffing, per-device rate limits and
|
|
389
|
+
broadcast fan-out all live in Rust.
|
|
390
|
+
- **Agents see it.** `dock_snapshot` is a tool that returns the live frame as
|
|
391
|
+
an image the model sees, over MCP too.
|
|
392
|
+
- **Watchers are the middle of the hub.** When the picture changes, the agent
|
|
393
|
+
looks, and its insight goes to every `/devices/dock/stream` subscriber, to a
|
|
394
|
+
webhook, and to the audit log. Spend is capped per hour.
|
|
395
|
+
- **Agents over WebSocket.** Any agent route accepts a socket: send a turn,
|
|
396
|
+
receive each step live, then the result.
|
|
397
|
+
- **A stdlib Python client** (`webcortex.client`): `DeviceConnection` for
|
|
398
|
+
drivers, `subscribe()` for consumers, `AgentSocket` for services.
|
|
399
|
+
|
|
400
|
+
`webcortex new site -t hub` scaffolds it. See [The device hub](docs/device-hub.md).
|
|
401
|
+
|
|
324
402
|
## Security defaults
|
|
325
403
|
|
|
326
404
|
Deny-by-default throughout; relaxing something costs a line, tightening it costs
|
|
@@ -404,7 +482,7 @@ tested, the known limits, and what v2 added to the surface.
|
|
|
404
482
|
|
|
405
483
|
## Status
|
|
406
484
|
|
|
407
|
-
v2.
|
|
485
|
+
v2.3.0. Working and tested: the manifest IR, router, native ops (static /
|
|
408
486
|
query / proxy / page / files / flow), the free-threaded Python bridge,
|
|
409
487
|
authentication and scopes, rate limiting, CORS, security headers, graceful
|
|
410
488
|
shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
|
|
@@ -412,8 +490,10 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
|
|
|
412
490
|
compaction, context providers, memory, flows, two providers (Anthropic and
|
|
413
491
|
OpenAI-compatible, which covers local models), prompt caching, the spend
|
|
414
492
|
ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
|
|
415
|
-
context pack and `evolve
|
|
416
|
-
|
|
493
|
+
context pack and `evolve`, images in and out of agent runs and MCP, actuators
|
|
494
|
+
and the emergency stop, the device hub (cameras, sensors, watchers, WebSocket
|
|
495
|
+
ingest and streams, agents over WebSocket). **427 tests** (132 Rust, 295 Python, including a
|
|
496
|
+
54-test adversarial suite and offline end-to-end suites that drive the whole
|
|
417
497
|
agent stack over HTTP), clippy clean.
|
|
418
498
|
|
|
419
499
|
Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
|
|
@@ -404,33 +404,57 @@ impl ModelProvider for OpenAiCompatProvider {
|
|
|
404
404
|
|
|
405
405
|
/// Translate one canonical (Anthropic-shaped) message into OpenAI messages.
|
|
406
406
|
/// A user turn holding tool results becomes one `tool` message per result.
|
|
407
|
+
///
|
|
408
|
+
/// OpenAI `tool` messages carry text only, so images a tool returned travel in
|
|
409
|
+
/// one `user` message after the tool messages, labelled with the call they
|
|
410
|
+
/// came from. Images sent with the input become `image_url` parts.
|
|
407
411
|
pub fn to_openai_messages(role: &str, content: &Value) -> Vec<Value> {
|
|
408
412
|
match (role, content) {
|
|
409
413
|
(_, Value::String(s)) => vec![json!({"role": role, "content": s})],
|
|
410
414
|
("user", Value::Array(blocks)) => {
|
|
411
415
|
let mut out = Vec::new();
|
|
412
416
|
let mut text = String::new();
|
|
417
|
+
let mut parts: Vec<Value> = Vec::new();
|
|
413
418
|
for b in blocks {
|
|
414
419
|
match b.get("type").and_then(|t| t.as_str()) {
|
|
415
|
-
Some("tool_result") =>
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
Some(Value::
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
420
|
+
Some("tool_result") => {
|
|
421
|
+
let id = b.get("tool_use_id").cloned().unwrap_or(Value::Null);
|
|
422
|
+
let (body, images) = match b.get("content") {
|
|
423
|
+
Some(Value::String(s)) => (s.clone(), Vec::new()),
|
|
424
|
+
Some(Value::Array(inner)) => (
|
|
425
|
+
crate::agent::message_text(&Value::Array(inner.clone())),
|
|
426
|
+
inner.iter().filter_map(super::vision::openai_part).collect(),
|
|
427
|
+
),
|
|
428
|
+
Some(other) => (other.to_string(), Vec::new()),
|
|
429
|
+
None => (String::new(), Vec::new()),
|
|
430
|
+
};
|
|
431
|
+
if !images.is_empty() {
|
|
432
|
+
parts.push(json!({"type": "text", "text": format!(
|
|
433
|
+
"Images returned by tool call {}:",
|
|
434
|
+
id.as_str().unwrap_or("?")
|
|
435
|
+
)}));
|
|
436
|
+
parts.extend(images);
|
|
437
|
+
}
|
|
438
|
+
out.push(json!({"role": "tool", "tool_call_id": id, "content": body}));
|
|
439
|
+
}
|
|
424
440
|
Some("text") => {
|
|
425
441
|
if let Some(t) = b.get("text").and_then(|t| t.as_str()) {
|
|
426
442
|
text.push_str(t);
|
|
427
443
|
}
|
|
428
444
|
}
|
|
445
|
+
Some("image") => parts.extend(super::vision::openai_part(b)),
|
|
429
446
|
_ => {}
|
|
430
447
|
}
|
|
431
448
|
}
|
|
432
|
-
if
|
|
433
|
-
|
|
449
|
+
if parts.is_empty() {
|
|
450
|
+
if !text.is_empty() {
|
|
451
|
+
out.push(json!({"role": "user", "content": text}));
|
|
452
|
+
}
|
|
453
|
+
} else {
|
|
454
|
+
if !text.is_empty() {
|
|
455
|
+
parts.push(json!({"type": "text", "text": text}));
|
|
456
|
+
}
|
|
457
|
+
out.push(json!({"role": "user", "content": parts}));
|
|
434
458
|
}
|
|
435
459
|
out
|
|
436
460
|
}
|
|
@@ -878,10 +902,20 @@ impl ModelProvider for FakeProvider {
|
|
|
878
902
|
};
|
|
879
903
|
|
|
880
904
|
let last = req.conversation.messages.last();
|
|
905
|
+
let is_tool_results = |c: &Value| {
|
|
906
|
+
c.as_array().is_some_and(|bs| {
|
|
907
|
+
bs.iter().any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result"))
|
|
908
|
+
})
|
|
909
|
+
};
|
|
881
910
|
let last_text = match last.map(|m| &m.content) {
|
|
882
911
|
Some(Value::String(s)) => Some(s.clone()),
|
|
912
|
+
// Input sent with images: the text block is the prompt.
|
|
913
|
+
Some(c @ Value::Array(_)) if !is_tool_results(c) => Some(crate::agent::message_text(c)),
|
|
883
914
|
_ => None,
|
|
884
915
|
};
|
|
916
|
+
// So a test can see that pixels reached the model.
|
|
917
|
+
let seen = last.map(|m| super::vision::count(&m.content)).unwrap_or(0);
|
|
918
|
+
let images_note = if seen > 0 { format!(" [saw {seen} image(s)]") } else { String::new() };
|
|
885
919
|
|
|
886
920
|
// A forced tool: structured output.
|
|
887
921
|
if let Some(forced) = req.force_tool {
|
|
@@ -914,7 +948,8 @@ impl ModelProvider for FakeProvider {
|
|
|
914
948
|
let summary = match last.map(|m| &m.content) {
|
|
915
949
|
Some(Value::Array(blocks)) => blocks
|
|
916
950
|
.iter()
|
|
917
|
-
.filter_map(|b| b.get("content")
|
|
951
|
+
.filter_map(|b| b.get("content"))
|
|
952
|
+
.map(|c| c.as_str().map(str::to_string).unwrap_or_else(|| crate::agent::message_text(c)))
|
|
918
953
|
.map(|s| {
|
|
919
954
|
let n = s.chars().count();
|
|
920
955
|
let tail: String = s.chars().skip(n.saturating_sub(160)).collect();
|
|
@@ -925,7 +960,7 @@ impl ModelProvider for FakeProvider {
|
|
|
925
960
|
_ => String::new(),
|
|
926
961
|
};
|
|
927
962
|
|
|
928
|
-
let reply = format!("{} says: {summary}", req.agent.name);
|
|
963
|
+
let reply = format!("{} says: {summary}{images_note}", req.agent.name);
|
|
929
964
|
return done(
|
|
930
965
|
reply.clone(),
|
|
931
966
|
Vec::new(),
|
|
@@ -964,7 +999,7 @@ impl ModelProvider for FakeProvider {
|
|
|
964
999
|
return done(String::new(), calls, Value::Array(raw), StopReason::ToolUse);
|
|
965
1000
|
}
|
|
966
1001
|
|
|
967
|
-
let reply = format!("{} echoes: {text}", req.agent.name);
|
|
1002
|
+
let reply = format!("{} echoes: {text}{images_note}", req.agent.name);
|
|
968
1003
|
done(
|
|
969
1004
|
reply.clone(),
|
|
970
1005
|
Vec::new(),
|
|
@@ -1105,6 +1140,27 @@ mod tests {
|
|
|
1105
1140
|
assert_eq!(out[1]["content"], "{\"n\":2}");
|
|
1106
1141
|
}
|
|
1107
1142
|
|
|
1143
|
+
#[test]
|
|
1144
|
+
fn images_reach_openai_as_image_url_parts_after_the_tool_messages() {
|
|
1145
|
+
let img = json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}});
|
|
1146
|
+
let content = json!([
|
|
1147
|
+
{"type": "tool_result", "tool_use_id": "c1", "is_error": false,
|
|
1148
|
+
"content": [{"type": "text", "text": "{\"frame\":\"[image 1]\"}"}, img]},
|
|
1149
|
+
]);
|
|
1150
|
+
let out = to_openai_messages("user", &content);
|
|
1151
|
+
assert_eq!(out.len(), 2, "tool message, then the images");
|
|
1152
|
+
assert_eq!(out[0]["role"], "tool");
|
|
1153
|
+
assert_eq!(out[0]["content"], "{\"frame\":\"[image 1]\"}");
|
|
1154
|
+
assert_eq!(out[1]["role"], "user");
|
|
1155
|
+
assert_eq!(out[1]["content"][1]["image_url"]["url"], "data:image/png;base64,AAAA");
|
|
1156
|
+
|
|
1157
|
+
let input = json!([img, {"type": "text", "text": "what is this?"}]);
|
|
1158
|
+
let out = to_openai_messages("user", &input);
|
|
1159
|
+
assert_eq!(out.len(), 1);
|
|
1160
|
+
assert_eq!(out[0]["content"][0]["type"], "image_url");
|
|
1161
|
+
assert_eq!(out[0]["content"][1]["text"], "what is this?");
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1108
1164
|
#[test]
|
|
1109
1165
|
fn canonical_assistant_tool_use_becomes_openai_tool_calls() {
|
|
1110
1166
|
let content = json!([
|