web-cortex-framework 2.1.1__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/Cargo.lock +48 -2
  2. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/Cargo.toml +4 -1
  3. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/PKG-INFO +84 -4
  4. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/README.md +83 -3
  5. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/Cargo.toml +1 -0
  6. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/provider.rs +70 -14
  7. web_cortex_framework-2.3.0/crates/webcortex-core/src/agent/vision.rs +348 -0
  8. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent.rs +249 -14
  9. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/app.rs +178 -0
  10. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/auth.rs +5 -0
  11. web_cortex_framework-2.3.0/crates/webcortex-core/src/devices.rs +718 -0
  12. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/lib.rs +2 -0
  13. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/manifest.rs +196 -0
  14. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/mcp.rs +28 -2
  15. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/server.rs +60 -2
  16. web_cortex_framework-2.3.0/crates/webcortex-core/src/ws.rs +341 -0
  17. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/src/behaviour.rs +2 -4
  18. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/pyproject.toml +1 -1
  19. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/__init__.py +3 -1
  20. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/_bridge.py +4 -0
  21. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/app.py +266 -0
  22. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/cli.py +64 -1
  23. web_cortex_framework-2.3.0/python/webcortex/client.py +292 -0
  24. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/contextpack.py +32 -3
  25. web_cortex_framework-2.3.0/python/webcortex/media.py +240 -0
  26. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/starters.py +295 -2
  27. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/LICENSE +0 -0
  28. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/registry.rs +0 -0
  29. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/agent/session.rs +0 -0
  30. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/audit.rs +0 -0
  31. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/bridge.rs +0 -0
  32. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/context.rs +0 -0
  33. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/db.rs +0 -0
  34. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/files.rs +0 -0
  35. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/flow.rs +0 -0
  36. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/http.rs +0 -0
  37. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/ledger.rs +0 -0
  38. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/middleware.rs +0 -0
  39. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/openapi.rs +0 -0
  40. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/router.rs +0 -0
  41. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/templates.rs +0 -0
  42. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-core/src/typegen.rs +0 -0
  43. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/Cargo.toml +0 -0
  44. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/crates/webcortex-py/src/lib.rs +0 -0
  45. {web_cortex_framework-2.1.1 → web_cortex_framework-2.3.0}/python/webcortex/schema.py +0 -0
@@ -286,6 +286,12 @@ dependencies = [
286
286
  "syn 2.0.119",
287
287
  ]
288
288
 
289
+ [[package]]
290
+ name = "data-encoding"
291
+ version = "2.11.1"
292
+ source = "registry+https://github.com/rust-lang/crates.io-index"
293
+ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
294
+
289
295
  [[package]]
290
296
  name = "der"
291
297
  version = "0.7.10"
@@ -1824,6 +1830,17 @@ dependencies = [
1824
1830
  "serde",
1825
1831
  ]
1826
1832
 
1833
+ [[package]]
1834
+ name = "sha1"
1835
+ version = "0.11.0"
1836
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1837
+ checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
1838
+ dependencies = [
1839
+ "cfg-if",
1840
+ "cpufeatures 0.3.0",
1841
+ "digest 0.11.3",
1842
+ ]
1843
+
1827
1844
  [[package]]
1828
1845
  name = "sha2"
1829
1846
  version = "0.10.9"
@@ -2304,6 +2321,18 @@ dependencies = [
2304
2321
  "tokio",
2305
2322
  ]
2306
2323
 
2324
+ [[package]]
2325
+ name = "tokio-tungstenite"
2326
+ version = "0.30.0"
2327
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2328
+ checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71"
2329
+ dependencies = [
2330
+ "futures-util",
2331
+ "log",
2332
+ "tokio",
2333
+ "tungstenite",
2334
+ ]
2335
+
2307
2336
  [[package]]
2308
2337
  name = "tokio-util"
2309
2338
  version = "0.7.19"
@@ -2431,6 +2460,22 @@ version = "0.2.5"
2431
2460
  source = "registry+https://github.com/rust-lang/crates.io-index"
2432
2461
  checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
2433
2462
 
2463
+ [[package]]
2464
+ name = "tungstenite"
2465
+ version = "0.30.0"
2466
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2467
+ checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
2468
+ dependencies = [
2469
+ "bytes",
2470
+ "data-encoding",
2471
+ "http",
2472
+ "httparse",
2473
+ "log",
2474
+ "rand 0.10.2",
2475
+ "sha1",
2476
+ "thiserror",
2477
+ ]
2478
+
2434
2479
  [[package]]
2435
2480
  name = "typenum"
2436
2481
  version = "1.20.1"
@@ -2616,7 +2661,7 @@ dependencies = [
2616
2661
 
2617
2662
  [[package]]
2618
2663
  name = "webcortex-core"
2619
- version = "2.1.1"
2664
+ version = "2.3.0"
2620
2665
  dependencies = [
2621
2666
  "base64 0.23.1",
2622
2667
  "bytes",
@@ -2636,6 +2681,7 @@ dependencies = [
2636
2681
  "sqlx",
2637
2682
  "subtle",
2638
2683
  "tokio",
2684
+ "tokio-tungstenite",
2639
2685
  "tracing",
2640
2686
  "tracing-subscriber",
2641
2687
  "uuid",
@@ -2643,7 +2689,7 @@ dependencies = [
2643
2689
 
2644
2690
  [[package]]
2645
2691
  name = "webcortex-py"
2646
- version = "2.1.1"
2692
+ version = "2.3.0"
2647
2693
  dependencies = [
2648
2694
  "bytes",
2649
2695
  "futures",
@@ -3,7 +3,7 @@ members = ["crates/webcortex-core", "crates/webcortex-py"]
3
3
  resolver = "3"
4
4
 
5
5
  [workspace.package]
6
- version = "2.1.1"
6
+ version = "2.3.0"
7
7
  edition = "2024"
8
8
  license = "Apache-2.0"
9
9
  rust-version = "1.85"
@@ -39,6 +39,9 @@ subtle = "2"
39
39
  base64 = "0.23"
40
40
  uuid = { version = "1", features = ["v4", "fast-rng"] }
41
41
  mime_guess = "2"
42
+ # WebSockets: devices stream frames in, subscribers and agents stream out. Only the
43
+ # server handshake is used; the HTTP upgrade itself is hyper's.
44
+ tokio-tungstenite = { version = "0.30", default-features = false, features = ["handshake"] }
42
45
 
43
46
  [profile.release]
44
47
  opt-level = 3
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: web-cortex-framework
3
- Version: 2.1.1
3
+ Version: 2.3.0
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: Apache Software License
@@ -349,6 +349,84 @@ invoke.
349
349
  **A full audit trail**, including refused calls, handoffs, compactions and
350
350
  approval decisions, at `GET /_webcortex/audit`.
351
351
 
352
+ ## Agents that see, machines that move
353
+
354
+ WebCortex embeds no vision model — it carries pixels to the model that has one,
355
+ and puts the guard rails a physical system needs around the tools that move it.
356
+
357
+ ```python
358
+ from webcortex import HTTPError, Image, WebCortex
359
+
360
+ app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
361
+
362
+ @app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
363
+ def camera() -> dict:
364
+ frame = cam.read() # an OpenCV / NumPy frame
365
+ return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
366
+
367
+ @app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
368
+ actuator=True, approval="required")
369
+ def move(joint: str, degrees: float) -> dict:
370
+ if not -90 <= degrees <= 90:
371
+ raise HTTPError(422, "out of range") # limits live in code, not prompts
372
+ return arm.move(joint, degrees)
373
+
374
+ app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
375
+ ```
376
+
377
+ - **The agent sees the frame.** A tool that returns a `webcortex.Image` —
378
+ anywhere in its result — hands the model the picture itself, over both wire
379
+ formats (Anthropic and OpenAI-compatible, so local vision models too) and over
380
+ MCP, where Claude Code and other clients receive it as image content. Agent
381
+ endpoints also take `"images": [...]` with the input.
382
+ - **Frames do not become a token leak.** Images are lifted out before
383
+ tool-result bounding, step records keep their size rather than their base64,
384
+ and only the last `max_images` stay in context as pixels.
385
+ - **An emergency stop that cannot be routed around.** `actuator=True` routes
386
+ refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
387
+ flow, MCP, or an approval granted after the stop. `webcortex halt` /
388
+ `webcortex release`, or `POST /_webcortex/halt` and `/release`.
389
+ - **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
390
+ an inspector agent and an operator agent, gated, scoped and booting halted.
391
+ Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
392
+
393
+ See [Vision and robotics](docs/vision-and-robotics.md).
394
+
395
+ ## The device hub: cameras in, insights out
396
+
397
+ WebCortex sits between devices and agents, and the Rust core does the work.
398
+ Cameras and sensors stream in, agents see what they send, and what the agents
399
+ conclude streams out to anyone subscribed.
400
+
401
+ ```python
402
+ app.camera("dock", max_fps=10) # pushed: browser, Pi, ESP32, curl
403
+ app.camera("lobby", source="http://192.168.1.40/snapshot.jpg") # pulled: any IP camera
404
+ app.sensor("dock_env") # telemetry only
405
+
406
+ app.agent("inspector", tools=["dock_snapshot", "dock_env_telemetry"], max_images=2)
407
+ app.watch("dock_watch", device="dock", agent="inspector", every=15,
408
+ input="Anything unsafe at the dock?", max_runs_per_hour=120,
409
+ webhook="https://ops.example.com/hooks/dock")
410
+ ```
411
+
412
+ - **Connect anything.** WebSocket (`/devices/dock/ws`: binary messages are
413
+ frames, text is telemetry), HTTP POST, or an IP camera's snapshot URL. Open
414
+ `/devices/dock/connect` on a phone or laptop and its camera becomes the
415
+ device.
416
+ - **No Python per frame.** Ring buffers, sniffing, per-device rate limits and
417
+ broadcast fan-out all live in Rust.
418
+ - **Agents see it.** `dock_snapshot` is a tool that returns the live frame as
419
+ an image the model sees, over MCP too.
420
+ - **Watchers are the middle of the hub.** When the picture changes, the agent
421
+ looks, and its insight goes to every `/devices/dock/stream` subscriber, to a
422
+ webhook, and to the audit log. Spend is capped per hour.
423
+ - **Agents over WebSocket.** Any agent route accepts a socket: send a turn,
424
+ receive each step live, then the result.
425
+ - **A stdlib Python client** (`webcortex.client`): `DeviceConnection` for
426
+ drivers, `subscribe()` for consumers, `AgentSocket` for services.
427
+
428
+ `webcortex new site -t hub` scaffolds it. See [The device hub](docs/device-hub.md).
429
+
352
430
  ## Security defaults
353
431
 
354
432
  Deny-by-default throughout; relaxing something costs a line, tightening it costs
@@ -432,7 +510,7 @@ tested, the known limits, and what v2 added to the surface.
432
510
 
433
511
  ## Status
434
512
 
435
- v2.1.1. Working and tested: the manifest IR, router, native ops (static /
513
+ v2.3.0. Working and tested: the manifest IR, router, native ops (static /
436
514
  query / proxy / page / files / flow), the free-threaded Python bridge,
437
515
  authentication and scopes, rate limiting, CORS, security headers, graceful
438
516
  shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
@@ -440,8 +518,10 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
440
518
  compaction, context providers, memory, flows, two providers (Anthropic and
441
519
  OpenAI-compatible, which covers local models), prompt caching, the spend
442
520
  ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
443
- context pack and `evolve`. **355 tests** (106 Rust, 249 Python, including a
444
- 54-test adversarial suite and an offline end-to-end suite that drives the whole
521
+ context pack and `evolve`, images in and out of agent runs and MCP, actuators
522
+ and the emergency stop, the device hub (cameras, sensors, watchers, WebSocket
523
+ ingest and streams, agents over WebSocket). **427 tests** (132 Rust, 295 Python, including a
524
+ 54-test adversarial suite and offline end-to-end suites that drive the whole
445
525
  agent stack over HTTP), clippy clean.
446
526
 
447
527
  Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
@@ -321,6 +321,84 @@ invoke.
321
321
  **A full audit trail**, including refused calls, handoffs, compactions and
322
322
  approval decisions, at `GET /_webcortex/audit`.
323
323
 
324
+ ## Agents that see, machines that move
325
+
326
+ WebCortex embeds no vision model — it carries pixels to the model that has one,
327
+ and puts the guard rails a physical system needs around the tools that move it.
328
+
329
+ ```python
330
+ from webcortex import HTTPError, Image, WebCortex
331
+
332
+ app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
333
+
334
+ @app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
335
+ def camera() -> dict:
336
+ frame = cam.read() # an OpenCV / NumPy frame
337
+ return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
338
+
339
+ @app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
340
+ actuator=True, approval="required")
341
+ def move(joint: str, degrees: float) -> dict:
342
+ if not -90 <= degrees <= 90:
343
+ raise HTTPError(422, "out of range") # limits live in code, not prompts
344
+ return arm.move(joint, degrees)
345
+
346
+ app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
347
+ ```
348
+
349
+ - **The agent sees the frame.** A tool that returns a `webcortex.Image` —
350
+ anywhere in its result — hands the model the picture itself, over both wire
351
+ formats (Anthropic and OpenAI-compatible, so local vision models too) and over
352
+ MCP, where Claude Code and other clients receive it as image content. Agent
353
+ endpoints also take `"images": [...]` with the input.
354
+ - **Frames do not become a token leak.** Images are lifted out before
355
+ tool-result bounding, step records keep their size rather than their base64,
356
+ and only the last `max_images` stay in context as pixels.
357
+ - **An emergency stop that cannot be routed around.** `actuator=True` routes
358
+ refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
359
+ flow, MCP, or an approval granted after the stop. `webcortex halt` /
360
+ `webcortex release`, or `POST /_webcortex/halt` and `/release`.
361
+ - **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
362
+ an inspector agent and an operator agent, gated, scoped and booting halted.
363
+ Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
364
+
365
+ See [Vision and robotics](docs/vision-and-robotics.md).
366
+
367
+ ## The device hub: cameras in, insights out
368
+
369
+ WebCortex sits between devices and agents, and the Rust core does the work.
370
+ Cameras and sensors stream in, agents see what they send, and what the agents
371
+ conclude streams out to anyone subscribed.
372
+
373
+ ```python
374
+ app.camera("dock", max_fps=10) # pushed: browser, Pi, ESP32, curl
375
+ app.camera("lobby", source="http://192.168.1.40/snapshot.jpg") # pulled: any IP camera
376
+ app.sensor("dock_env") # telemetry only
377
+
378
+ app.agent("inspector", tools=["dock_snapshot", "dock_env_telemetry"], max_images=2)
379
+ app.watch("dock_watch", device="dock", agent="inspector", every=15,
380
+ input="Anything unsafe at the dock?", max_runs_per_hour=120,
381
+ webhook="https://ops.example.com/hooks/dock")
382
+ ```
383
+
384
+ - **Connect anything.** WebSocket (`/devices/dock/ws`: binary messages are
385
+ frames, text is telemetry), HTTP POST, or an IP camera's snapshot URL. Open
386
+ `/devices/dock/connect` on a phone or laptop and its camera becomes the
387
+ device.
388
+ - **No Python per frame.** Ring buffers, sniffing, per-device rate limits and
389
+ broadcast fan-out all live in Rust.
390
+ - **Agents see it.** `dock_snapshot` is a tool that returns the live frame as
391
+ an image the model sees, over MCP too.
392
+ - **Watchers are the middle of the hub.** When the picture changes, the agent
393
+ looks, and its insight goes to every `/devices/dock/stream` subscriber, to a
394
+ webhook, and to the audit log. Spend is capped per hour.
395
+ - **Agents over WebSocket.** Any agent route accepts a socket: send a turn,
396
+ receive each step live, then the result.
397
+ - **A stdlib Python client** (`webcortex.client`): `DeviceConnection` for
398
+ drivers, `subscribe()` for consumers, `AgentSocket` for services.
399
+
400
+ `webcortex new site -t hub` scaffolds it. See [The device hub](docs/device-hub.md).
401
+
324
402
  ## Security defaults
325
403
 
326
404
  Deny-by-default throughout; relaxing something costs a line, tightening it costs
@@ -404,7 +482,7 @@ tested, the known limits, and what v2 added to the surface.
404
482
 
405
483
  ## Status
406
484
 
407
- v2.1.1. Working and tested: the manifest IR, router, native ops (static /
485
+ v2.3.0. Working and tested: the manifest IR, router, native ops (static /
408
486
  query / proxy / page / files / flow), the free-threaded Python bridge,
409
487
  authentication and scopes, rate limiting, CORS, security headers, graceful
410
488
  shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
@@ -412,8 +490,10 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
412
490
  compaction, context providers, memory, flows, two providers (Anthropic and
413
491
  OpenAI-compatible, which covers local models), prompt caching, the spend
414
492
  ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
415
- context pack and `evolve`. **355 tests** (106 Rust, 249 Python, including a
416
- 54-test adversarial suite and an offline end-to-end suite that drives the whole
493
+ context pack and `evolve`, images in and out of agent runs and MCP, actuators
494
+ and the emergency stop, the device hub (cameras, sensors, watchers, WebSocket
495
+ ingest and streams, agents over WebSocket). **427 tests** (132 Rust, 295 Python, including a
496
+ 54-test adversarial suite and offline end-to-end suites that drive the whole
417
497
  agent stack over HTTP), clippy clean.
418
498
 
419
499
  Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
@@ -32,6 +32,7 @@ subtle.workspace = true
32
32
  base64.workspace = true
33
33
  uuid.workspace = true
34
34
  mime_guess.workspace = true
35
+ tokio-tungstenite.workspace = true
35
36
 
36
37
  [features]
37
38
  default = ["sqlite"]
@@ -404,33 +404,57 @@ impl ModelProvider for OpenAiCompatProvider {
404
404
 
405
405
  /// Translate one canonical (Anthropic-shaped) message into OpenAI messages.
406
406
  /// A user turn holding tool results becomes one `tool` message per result.
407
+ ///
408
+ /// OpenAI `tool` messages carry text only, so images a tool returned travel in
409
+ /// one `user` message after the tool messages, labelled with the call they
410
+ /// came from. Images sent with the input become `image_url` parts.
407
411
  pub fn to_openai_messages(role: &str, content: &Value) -> Vec<Value> {
408
412
  match (role, content) {
409
413
  (_, Value::String(s)) => vec![json!({"role": role, "content": s})],
410
414
  ("user", Value::Array(blocks)) => {
411
415
  let mut out = Vec::new();
412
416
  let mut text = String::new();
417
+ let mut parts: Vec<Value> = Vec::new();
413
418
  for b in blocks {
414
419
  match b.get("type").and_then(|t| t.as_str()) {
415
- Some("tool_result") => out.push(json!({
416
- "role": "tool",
417
- "tool_call_id": b.get("tool_use_id").cloned().unwrap_or(Value::Null),
418
- "content": match b.get("content") {
419
- Some(Value::String(s)) => s.clone(),
420
- Some(other) => other.to_string(),
421
- None => String::new(),
422
- },
423
- })),
420
+ Some("tool_result") => {
421
+ let id = b.get("tool_use_id").cloned().unwrap_or(Value::Null);
422
+ let (body, images) = match b.get("content") {
423
+ Some(Value::String(s)) => (s.clone(), Vec::new()),
424
+ Some(Value::Array(inner)) => (
425
+ crate::agent::message_text(&Value::Array(inner.clone())),
426
+ inner.iter().filter_map(super::vision::openai_part).collect(),
427
+ ),
428
+ Some(other) => (other.to_string(), Vec::new()),
429
+ None => (String::new(), Vec::new()),
430
+ };
431
+ if !images.is_empty() {
432
+ parts.push(json!({"type": "text", "text": format!(
433
+ "Images returned by tool call {}:",
434
+ id.as_str().unwrap_or("?")
435
+ )}));
436
+ parts.extend(images);
437
+ }
438
+ out.push(json!({"role": "tool", "tool_call_id": id, "content": body}));
439
+ }
424
440
  Some("text") => {
425
441
  if let Some(t) = b.get("text").and_then(|t| t.as_str()) {
426
442
  text.push_str(t);
427
443
  }
428
444
  }
445
+ Some("image") => parts.extend(super::vision::openai_part(b)),
429
446
  _ => {}
430
447
  }
431
448
  }
432
- if !text.is_empty() {
433
- out.push(json!({"role": "user", "content": text}));
449
+ if parts.is_empty() {
450
+ if !text.is_empty() {
451
+ out.push(json!({"role": "user", "content": text}));
452
+ }
453
+ } else {
454
+ if !text.is_empty() {
455
+ parts.push(json!({"type": "text", "text": text}));
456
+ }
457
+ out.push(json!({"role": "user", "content": parts}));
434
458
  }
435
459
  out
436
460
  }
@@ -878,10 +902,20 @@ impl ModelProvider for FakeProvider {
878
902
  };
879
903
 
880
904
  let last = req.conversation.messages.last();
905
+ let is_tool_results = |c: &Value| {
906
+ c.as_array().is_some_and(|bs| {
907
+ bs.iter().any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result"))
908
+ })
909
+ };
881
910
  let last_text = match last.map(|m| &m.content) {
882
911
  Some(Value::String(s)) => Some(s.clone()),
912
+ // Input sent with images: the text block is the prompt.
913
+ Some(c @ Value::Array(_)) if !is_tool_results(c) => Some(crate::agent::message_text(c)),
883
914
  _ => None,
884
915
  };
916
+ // So a test can see that pixels reached the model.
917
+ let seen = last.map(|m| super::vision::count(&m.content)).unwrap_or(0);
918
+ let images_note = if seen > 0 { format!(" [saw {seen} image(s)]") } else { String::new() };
885
919
 
886
920
  // A forced tool: structured output.
887
921
  if let Some(forced) = req.force_tool {
@@ -914,7 +948,8 @@ impl ModelProvider for FakeProvider {
914
948
  let summary = match last.map(|m| &m.content) {
915
949
  Some(Value::Array(blocks)) => blocks
916
950
  .iter()
917
- .filter_map(|b| b.get("content").and_then(|c| c.as_str()))
951
+ .filter_map(|b| b.get("content"))
952
+ .map(|c| c.as_str().map(str::to_string).unwrap_or_else(|| crate::agent::message_text(c)))
918
953
  .map(|s| {
919
954
  let n = s.chars().count();
920
955
  let tail: String = s.chars().skip(n.saturating_sub(160)).collect();
@@ -925,7 +960,7 @@ impl ModelProvider for FakeProvider {
925
960
  _ => String::new(),
926
961
  };
927
962
 
928
- let reply = format!("{} says: {summary}", req.agent.name);
963
+ let reply = format!("{} says: {summary}{images_note}", req.agent.name);
929
964
  return done(
930
965
  reply.clone(),
931
966
  Vec::new(),
@@ -964,7 +999,7 @@ impl ModelProvider for FakeProvider {
964
999
  return done(String::new(), calls, Value::Array(raw), StopReason::ToolUse);
965
1000
  }
966
1001
 
967
- let reply = format!("{} echoes: {text}", req.agent.name);
1002
+ let reply = format!("{} echoes: {text}{images_note}", req.agent.name);
968
1003
  done(
969
1004
  reply.clone(),
970
1005
  Vec::new(),
@@ -1105,6 +1140,27 @@ mod tests {
1105
1140
  assert_eq!(out[1]["content"], "{\"n\":2}");
1106
1141
  }
1107
1142
 
1143
+ #[test]
1144
+ fn images_reach_openai_as_image_url_parts_after_the_tool_messages() {
1145
+ let img = json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}});
1146
+ let content = json!([
1147
+ {"type": "tool_result", "tool_use_id": "c1", "is_error": false,
1148
+ "content": [{"type": "text", "text": "{\"frame\":\"[image 1]\"}"}, img]},
1149
+ ]);
1150
+ let out = to_openai_messages("user", &content);
1151
+ assert_eq!(out.len(), 2, "tool message, then the images");
1152
+ assert_eq!(out[0]["role"], "tool");
1153
+ assert_eq!(out[0]["content"], "{\"frame\":\"[image 1]\"}");
1154
+ assert_eq!(out[1]["role"], "user");
1155
+ assert_eq!(out[1]["content"][1]["image_url"]["url"], "data:image/png;base64,AAAA");
1156
+
1157
+ let input = json!([img, {"type": "text", "text": "what is this?"}]);
1158
+ let out = to_openai_messages("user", &input);
1159
+ assert_eq!(out.len(), 1);
1160
+ assert_eq!(out[0]["content"][0]["type"], "image_url");
1161
+ assert_eq!(out[0]["content"][1]["text"], "what is this?");
1162
+ }
1163
+
1108
1164
  #[test]
1109
1165
  fn canonical_assistant_tool_use_becomes_openai_tool_calls() {
1110
1166
  let content = json!([