web-cortex-framework 2.1.1__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/Cargo.lock +2 -2
  2. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/Cargo.toml +1 -1
  3. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/PKG-INFO +48 -4
  4. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/README.md +47 -3
  5. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/provider.rs +70 -14
  6. web_cortex_framework-2.2.0/crates/webcortex-core/src/agent/vision.rs +348 -0
  7. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent.rs +222 -14
  8. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/app.rs +70 -0
  9. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/manifest.rs +19 -0
  10. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/mcp.rs +28 -2
  11. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/server.rs +24 -0
  12. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/src/behaviour.rs +2 -4
  13. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/pyproject.toml +1 -1
  14. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/__init__.py +3 -1
  15. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/_bridge.py +4 -0
  16. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/app.py +74 -0
  17. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/cli.py +55 -1
  18. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/contextpack.py +18 -3
  19. web_cortex_framework-2.2.0/python/webcortex/media.py +240 -0
  20. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/starters.py +206 -2
  21. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/LICENSE +0 -0
  22. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/Cargo.toml +0 -0
  23. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/registry.rs +0 -0
  24. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/agent/session.rs +0 -0
  25. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/audit.rs +0 -0
  26. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/auth.rs +0 -0
  27. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/bridge.rs +0 -0
  28. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/context.rs +0 -0
  29. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/db.rs +0 -0
  30. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/files.rs +0 -0
  31. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/flow.rs +0 -0
  32. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/http.rs +0 -0
  33. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/ledger.rs +0 -0
  34. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/lib.rs +0 -0
  35. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/middleware.rs +0 -0
  36. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/openapi.rs +0 -0
  37. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/router.rs +0 -0
  38. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/templates.rs +0 -0
  39. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-core/src/typegen.rs +0 -0
  40. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/Cargo.toml +0 -0
  41. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/crates/webcortex-py/src/lib.rs +0 -0
  42. {web_cortex_framework-2.1.1 → web_cortex_framework-2.2.0}/python/webcortex/schema.py +0 -0
@@ -2616,7 +2616,7 @@ dependencies = [
2616
2616
 
2617
2617
  [[package]]
2618
2618
  name = "webcortex-core"
2619
- version = "2.1.1"
2619
+ version = "2.2.0"
2620
2620
  dependencies = [
2621
2621
  "base64 0.23.1",
2622
2622
  "bytes",
@@ -2643,7 +2643,7 @@ dependencies = [
2643
2643
 
2644
2644
  [[package]]
2645
2645
  name = "webcortex-py"
2646
- version = "2.1.1"
2646
+ version = "2.2.0"
2647
2647
  dependencies = [
2648
2648
  "bytes",
2649
2649
  "futures",
@@ -3,7 +3,7 @@ members = ["crates/webcortex-core", "crates/webcortex-py"]
3
3
  resolver = "3"
4
4
 
5
5
  [workspace.package]
6
- version = "2.1.1"
6
+ version = "2.2.0"
7
7
  edition = "2024"
8
8
  license = "Apache-2.0"
9
9
  rust-version = "1.85"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: web-cortex-framework
3
- Version: 2.1.1
3
+ Version: 2.2.0
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: Apache Software License
@@ -349,6 +349,49 @@ invoke.
349
349
  **A full audit trail**, including refused calls, handoffs, compactions and
350
350
  approval decisions, at `GET /_webcortex/audit`.
351
351
 
352
+ ## Agents that see, machines that move
353
+
354
+ WebCortex embeds no vision model — it carries pixels to the model that has one,
355
+ and puts the guard rails a physical system needs around the tools that move it.
356
+
357
+ ```python
358
+ from webcortex import HTTPError, Image, WebCortex
359
+
360
+ app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
361
+
362
+ @app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
363
+ def camera() -> dict:
364
+ frame = cam.read() # an OpenCV / NumPy frame
365
+ return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
366
+
367
+ @app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
368
+ actuator=True, approval="required")
369
+ def move(joint: str, degrees: float) -> dict:
370
+ if not -90 <= degrees <= 90:
371
+ raise HTTPError(422, "out of range") # limits live in code, not prompts
372
+ return arm.move(joint, degrees)
373
+
374
+ app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
375
+ ```
376
+
377
+ - **The agent sees the frame.** A tool that returns a `webcortex.Image` —
378
+ anywhere in its result — hands the model the picture itself, over both wire
379
+ formats (Anthropic and OpenAI-compatible, so local vision models too) and over
380
+ MCP, where Claude Code and other clients receive it as image content. Agent
381
+ endpoints also take `"images": [...]` with the input.
382
+ - **Frames do not become a token leak.** Images are lifted out before
383
+ tool-result bounding, step records keep their size rather than their base64,
384
+ and only the last `max_images` stay in context as pixels.
385
+ - **An emergency stop that cannot be routed around.** `actuator=True` routes
386
+ refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
387
+ flow, MCP, or an approval granted after the stop. `webcortex halt` /
388
+ `webcortex release`, or `POST /_webcortex/halt` and `/release`.
389
+ - **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
390
+ an inspector agent and an operator agent, gated, scoped and booting halted.
391
+ Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
392
+
393
+ See [Vision and robotics](docs/vision-and-robotics.md).
394
+
352
395
  ## Security defaults
353
396
 
354
397
  Deny-by-default throughout; relaxing something costs a line, tightening it costs
@@ -432,7 +475,7 @@ tested, the known limits, and what v2 added to the surface.
432
475
 
433
476
  ## Status
434
477
 
435
- v2.1.1. Working and tested: the manifest IR, router, native ops (static /
478
+ v2.2.0. Working and tested: the manifest IR, router, native ops (static /
436
479
  query / proxy / page / files / flow), the free-threaded Python bridge,
437
480
  authentication and scopes, rate limiting, CORS, security headers, graceful
438
481
  shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
@@ -440,8 +483,9 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
440
483
  compaction, context providers, memory, flows, two providers (Anthropic and
441
484
  OpenAI-compatible, which covers local models), prompt caching, the spend
442
485
  ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
443
- context pack and `evolve`. **355 tests** (106 Rust, 249 Python, including a
444
- 54-test adversarial suite and an offline end-to-end suite that drives the whole
486
+ context pack and `evolve`, images in and out of agent runs and MCP, actuators
487
+ and the emergency stop. **403 tests** (124 Rust, 279 Python, including a
488
+ 54-test adversarial suite and offline end-to-end suites that drive the whole
445
489
  agent stack over HTTP), clippy clean.
446
490
 
447
491
  Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
@@ -321,6 +321,49 @@ invoke.
321
321
  **A full audit trail**, including refused calls, handoffs, compactions and
322
322
  approval decisions, at `GET /_webcortex/audit`.
323
323
 
324
+ ## Agents that see, machines that move
325
+
326
+ WebCortex embeds no vision model — it carries pixels to the model that has one,
327
+ and puts the guard rails a physical system needs around the tools that move it.
328
+
329
+ ```python
330
+ from webcortex import HTTPError, Image, WebCortex
331
+
332
+ app = WebCortex("cell", start_halted=True) # boots with the e-stop engaged
333
+
334
+ @app.get("/camera", tool=True, tool_name="camera", scopes=["observe"])
335
+ def camera() -> dict:
336
+ frame = cam.read() # an OpenCV / NumPy frame
337
+ return {"frame": Image.from_array(frame, bgr=True), "exposure_ms": 8}
338
+
339
+ @app.post("/arm/move", tool=True, tool_name="move_arm", scopes=["operate"],
340
+ actuator=True, approval="required")
341
+ def move(joint: str, degrees: float) -> dict:
342
+ if not -90 <= degrees <= 90:
343
+ raise HTTPError(422, "out of range") # limits live in code, not prompts
344
+ return arm.move(joint, degrees)
345
+
346
+ app.agent("operator", tools=["camera", "move_arm"], scopes=["observe", "operate"], max_images=3)
347
+ ```
348
+
349
+ - **The agent sees the frame.** A tool that returns a `webcortex.Image` —
350
+ anywhere in its result — hands the model the picture itself, over both wire
351
+ formats (Anthropic and OpenAI-compatible, so local vision models too) and over
352
+ MCP, where Claude Code and other clients receive it as image content. Agent
353
+ endpoints also take `"images": [...]` with the input.
354
+ - **Frames do not become a token leak.** Images are lifted out before
355
+ tool-result bounding, step records keep their size rather than their base64,
356
+ and only the last `max_images` stay in context as pixels.
357
+ - **An emergency stop that cannot be routed around.** `actuator=True` routes
358
+ refuse with `423` while halted — over HTTP, from an agent, a behaviour, a
359
+ flow, MCP, or an approval granted after the stop. `webcortex halt` /
360
+ `webcortex release`, or `POST /_webcortex/halt` and `/release`.
361
+ - **`webcortex new cell -t robotics`** scaffolds a simulated arm and camera with
362
+ an inspector agent and an operator agent, gated, scoped and booting halted.
363
+ Swap the `Cell` class for your driver — serial, ROS 2, Modbus, a vendor SDK.
364
+
365
+ See [Vision and robotics](docs/vision-and-robotics.md).
366
+
324
367
  ## Security defaults
325
368
 
326
369
  Deny-by-default throughout; relaxing something costs a line, tightening it costs
@@ -404,7 +447,7 @@ tested, the known limits, and what v2 added to the surface.
404
447
 
405
448
  ## Status
406
449
 
407
- v2.1.1. Working and tested: the manifest IR, router, native ops (static /
450
+ v2.2.0. Working and tested: the manifest IR, router, native ops (static /
408
451
  query / proxy / page / files / flow), the free-threaded Python bridge,
409
452
  authentication and scopes, rate limiting, CORS, security headers, graceful
410
453
  shutdown, Behaviours with concurrent leaves, the agent runtime with handoffs,
@@ -412,8 +455,9 @@ sessions, resumable approval gates, composing budgets, tool-result bounding and
412
455
  compaction, context providers, memory, flows, two providers (Anthropic and
413
456
  OpenAI-compatible, which covers local models), prompt caching, the spend
414
457
  ledger, the audit trail, OpenAPI, the MCP server, TypeScript generation, the
415
- context pack and `evolve`. **355 tests** (106 Rust, 249 Python, including a
416
- 54-test adversarial suite and an offline end-to-end suite that drives the whole
458
+ context pack and `evolve`, images in and out of agent runs and MCP, actuators
459
+ and the emergency stop. **403 tests** (124 Rust, 279 Python, including a
460
+ 54-test adversarial suite and offline end-to-end suites that drive the whole
417
461
  agent stack over HTTP), clippy clean.
418
462
 
419
463
  Not yet: Postgres, token-level SSE streaming, durable agent runs that survive a
@@ -404,33 +404,57 @@ impl ModelProvider for OpenAiCompatProvider {
404
404
 
405
405
  /// Translate one canonical (Anthropic-shaped) message into OpenAI messages.
406
406
  /// A user turn holding tool results becomes one `tool` message per result.
407
+ ///
408
+ /// OpenAI `tool` messages carry text only, so images a tool returned travel in
409
+ /// one `user` message after the tool messages, labelled with the call they
410
+ /// came from. Images sent with the input become `image_url` parts.
407
411
  pub fn to_openai_messages(role: &str, content: &Value) -> Vec<Value> {
408
412
  match (role, content) {
409
413
  (_, Value::String(s)) => vec![json!({"role": role, "content": s})],
410
414
  ("user", Value::Array(blocks)) => {
411
415
  let mut out = Vec::new();
412
416
  let mut text = String::new();
417
+ let mut parts: Vec<Value> = Vec::new();
413
418
  for b in blocks {
414
419
  match b.get("type").and_then(|t| t.as_str()) {
415
- Some("tool_result") => out.push(json!({
416
- "role": "tool",
417
- "tool_call_id": b.get("tool_use_id").cloned().unwrap_or(Value::Null),
418
- "content": match b.get("content") {
419
- Some(Value::String(s)) => s.clone(),
420
- Some(other) => other.to_string(),
421
- None => String::new(),
422
- },
423
- })),
420
+ Some("tool_result") => {
421
+ let id = b.get("tool_use_id").cloned().unwrap_or(Value::Null);
422
+ let (body, images) = match b.get("content") {
423
+ Some(Value::String(s)) => (s.clone(), Vec::new()),
424
+ Some(Value::Array(inner)) => (
425
+ crate::agent::message_text(&Value::Array(inner.clone())),
426
+ inner.iter().filter_map(super::vision::openai_part).collect(),
427
+ ),
428
+ Some(other) => (other.to_string(), Vec::new()),
429
+ None => (String::new(), Vec::new()),
430
+ };
431
+ if !images.is_empty() {
432
+ parts.push(json!({"type": "text", "text": format!(
433
+ "Images returned by tool call {}:",
434
+ id.as_str().unwrap_or("?")
435
+ )}));
436
+ parts.extend(images);
437
+ }
438
+ out.push(json!({"role": "tool", "tool_call_id": id, "content": body}));
439
+ }
424
440
  Some("text") => {
425
441
  if let Some(t) = b.get("text").and_then(|t| t.as_str()) {
426
442
  text.push_str(t);
427
443
  }
428
444
  }
445
+ Some("image") => parts.extend(super::vision::openai_part(b)),
429
446
  _ => {}
430
447
  }
431
448
  }
432
- if !text.is_empty() {
433
- out.push(json!({"role": "user", "content": text}));
449
+ if parts.is_empty() {
450
+ if !text.is_empty() {
451
+ out.push(json!({"role": "user", "content": text}));
452
+ }
453
+ } else {
454
+ if !text.is_empty() {
455
+ parts.push(json!({"type": "text", "text": text}));
456
+ }
457
+ out.push(json!({"role": "user", "content": parts}));
434
458
  }
435
459
  out
436
460
  }
@@ -878,10 +902,20 @@ impl ModelProvider for FakeProvider {
878
902
  };
879
903
 
880
904
  let last = req.conversation.messages.last();
905
+ let is_tool_results = |c: &Value| {
906
+ c.as_array().is_some_and(|bs| {
907
+ bs.iter().any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result"))
908
+ })
909
+ };
881
910
  let last_text = match last.map(|m| &m.content) {
882
911
  Some(Value::String(s)) => Some(s.clone()),
912
+ // Input sent with images: the text block is the prompt.
913
+ Some(c @ Value::Array(_)) if !is_tool_results(c) => Some(crate::agent::message_text(c)),
883
914
  _ => None,
884
915
  };
916
+ // So a test can see that pixels reached the model.
917
+ let seen = last.map(|m| super::vision::count(&m.content)).unwrap_or(0);
918
+ let images_note = if seen > 0 { format!(" [saw {seen} image(s)]") } else { String::new() };
885
919
 
886
920
  // A forced tool: structured output.
887
921
  if let Some(forced) = req.force_tool {
@@ -914,7 +948,8 @@ impl ModelProvider for FakeProvider {
914
948
  let summary = match last.map(|m| &m.content) {
915
949
  Some(Value::Array(blocks)) => blocks
916
950
  .iter()
917
- .filter_map(|b| b.get("content").and_then(|c| c.as_str()))
951
+ .filter_map(|b| b.get("content"))
952
+ .map(|c| c.as_str().map(str::to_string).unwrap_or_else(|| crate::agent::message_text(c)))
918
953
  .map(|s| {
919
954
  let n = s.chars().count();
920
955
  let tail: String = s.chars().skip(n.saturating_sub(160)).collect();
@@ -925,7 +960,7 @@ impl ModelProvider for FakeProvider {
925
960
  _ => String::new(),
926
961
  };
927
962
 
928
- let reply = format!("{} says: {summary}", req.agent.name);
963
+ let reply = format!("{} says: {summary}{images_note}", req.agent.name);
929
964
  return done(
930
965
  reply.clone(),
931
966
  Vec::new(),
@@ -964,7 +999,7 @@ impl ModelProvider for FakeProvider {
964
999
  return done(String::new(), calls, Value::Array(raw), StopReason::ToolUse);
965
1000
  }
966
1001
 
967
- let reply = format!("{} echoes: {text}", req.agent.name);
1002
+ let reply = format!("{} echoes: {text}{images_note}", req.agent.name);
968
1003
  done(
969
1004
  reply.clone(),
970
1005
  Vec::new(),
@@ -1105,6 +1140,27 @@ mod tests {
1105
1140
  assert_eq!(out[1]["content"], "{\"n\":2}");
1106
1141
  }
1107
1142
 
1143
+ #[test]
1144
+ fn images_reach_openai_as_image_url_parts_after_the_tool_messages() {
1145
+ let img = json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}});
1146
+ let content = json!([
1147
+ {"type": "tool_result", "tool_use_id": "c1", "is_error": false,
1148
+ "content": [{"type": "text", "text": "{\"frame\":\"[image 1]\"}"}, img]},
1149
+ ]);
1150
+ let out = to_openai_messages("user", &content);
1151
+ assert_eq!(out.len(), 2, "tool message, then the images");
1152
+ assert_eq!(out[0]["role"], "tool");
1153
+ assert_eq!(out[0]["content"], "{\"frame\":\"[image 1]\"}");
1154
+ assert_eq!(out[1]["role"], "user");
1155
+ assert_eq!(out[1]["content"][1]["image_url"]["url"], "data:image/png;base64,AAAA");
1156
+
1157
+ let input = json!([img, {"type": "text", "text": "what is this?"}]);
1158
+ let out = to_openai_messages("user", &input);
1159
+ assert_eq!(out.len(), 1);
1160
+ assert_eq!(out[0]["content"][0]["type"], "image_url");
1161
+ assert_eq!(out[0]["content"][1]["text"], "what is this?");
1162
+ }
1163
+
1108
1164
  #[test]
1109
1165
  fn canonical_assistant_tool_use_becomes_openai_tool_calls() {
1110
1166
  let content = json!([
@@ -0,0 +1,348 @@
1
+ //! Images in and out of a run.
2
+ //!
3
+ //! WebCortex does not run vision models — that would be embedded inference.
4
+ //! It carries pixels to the model that does: an image given with the request,
5
+ //! or a frame a tool returned, becomes an image block in the conversation.
6
+ //!
7
+ //! A tool marks an image in its JSON result with a one-key object,
8
+ //! `{"$image": {"media_type": "image/png", "data": "<base64>"}}` or
9
+ //! `{"$image": {"url": "https://…"}}` — which is what `webcortex.Image`
10
+ //! serialises to. The runtime lifts each one out of the result, leaves a short
11
+ //! placeholder in its place, and attaches the image beside the text, so the
12
+ //! model sees the picture and the step record keeps only its size.
13
+ //!
14
+ //! Images are expensive context: every step replays the conversation, and a
15
+ //! camera tool called in a loop would resend every frame it ever took.
16
+ //! [`prune`] keeps the most recent few and turns the rest into a line of text.
17
+
18
+ use base64::Engine;
19
+ use serde_json::{Value, json};
20
+
21
+ /// The key that marks an image inside a tool result or a request.
22
+ pub const IMAGE_KEY: &str = "$image";
23
+
24
+ /// Media types every supported provider accepts.
25
+ pub const MEDIA_TYPES: [&str; 4] = ["image/png", "image/jpeg", "image/gif", "image/webp"];
26
+
27
+ /// Decoded size ceiling for one image. Providers reject larger ones anyway;
28
+ /// refusing here gives a clearer error and keeps them out of session memory.
29
+ pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
30
+
31
+ /// Images accepted with one request, and lifted out of one tool result.
32
+ pub const MAX_IMAGES: usize = 16;
33
+
34
+ /// Turn one image spec into a canonical (Anthropic-shaped) image block.
35
+ ///
36
+ /// Accepts `{"media_type", "data"}`, `{"url"}`, or either wrapped in
37
+ /// `{"$image": …}`.
38
+ pub fn image_block(spec: &Value) -> Result<Value, String> {
39
+ let spec = spec.get(IMAGE_KEY).unwrap_or(spec);
40
+ if let Some(url) = spec.get("url").and_then(|u| u.as_str()) {
41
+ if !(url.starts_with("https://") || url.starts_with("http://")) {
42
+ return Err(format!("image url must be http(s): {url:?}"));
43
+ }
44
+ return Ok(json!({"type": "image", "source": {"type": "url", "url": url}}));
45
+ }
46
+ let data = spec
47
+ .get("data")
48
+ .and_then(|d| d.as_str())
49
+ .ok_or("an image needs base64 'data' with a 'media_type', or a 'url'")?;
50
+ let media_type = spec
51
+ .get("media_type")
52
+ .and_then(|m| m.as_str())
53
+ .ok_or("an image with 'data' needs a 'media_type'")?;
54
+ if !MEDIA_TYPES.contains(&media_type) {
55
+ return Err(format!(
56
+ "unsupported image media_type {media_type:?}; use one of {}",
57
+ MEDIA_TYPES.join(", ")
58
+ ));
59
+ }
60
+ let bytes = base64::engine::general_purpose::STANDARD
61
+ .decode(data)
62
+ .map_err(|e| format!("image data is not valid base64: {e}"))?;
63
+ if bytes.is_empty() {
64
+ return Err("image data is empty".into());
65
+ }
66
+ if bytes.len() > MAX_IMAGE_BYTES {
67
+ return Err(format!(
68
+ "image is {} bytes; the limit is {MAX_IMAGE_BYTES}",
69
+ bytes.len()
70
+ ));
71
+ }
72
+ Ok(json!({
73
+ "type": "image",
74
+ "source": {"type": "base64", "media_type": media_type, "data": data},
75
+ }))
76
+ }
77
+
78
+ /// The `images` field of an agent request, as canonical blocks.
79
+ pub fn input_images(value: Option<&Value>) -> Result<Vec<Value>, String> {
80
+ let Some(value) = value else { return Ok(Vec::new()) };
81
+ if value.is_null() {
82
+ return Ok(Vec::new());
83
+ }
84
+ let items = value.as_array().ok_or("'images' must be a list")?;
85
+ if items.len() > MAX_IMAGES {
86
+ return Err(format!("at most {MAX_IMAGES} images per request ({} given)", items.len()));
87
+ }
88
+ items
89
+ .iter()
90
+ .enumerate()
91
+ .map(|(i, v)| image_block(v).map_err(|e| format!("images[{i}]: {e}")))
92
+ .collect()
93
+ }
94
+
95
+ /// True for a `{"$image": …}` marker object.
96
+ fn is_marker(value: &Value) -> bool {
97
+ value.as_object().is_some_and(|o| o.len() == 1 && o.contains_key(IMAGE_KEY))
98
+ }
99
+
100
+ /// Decoded size of a base64 string, without decoding it.
101
+ fn decoded_len(data: &str) -> usize {
102
+ let pad = data.bytes().rev().take_while(|&b| b == b'=').count();
103
+ (data.len() / 4 * 3).saturating_sub(pad)
104
+ }
105
+
106
+ /// A one-line description of an image block, for placeholders and transcripts.
107
+ pub fn describe(block: &Value) -> String {
108
+ let source = block.get("source").cloned().unwrap_or(Value::Null);
109
+ match source.get("type").and_then(|t| t.as_str()) {
110
+ Some("url") => format!("image at {}", source.get("url").and_then(|u| u.as_str()).unwrap_or("?")),
111
+ _ => format!(
112
+ "{}, {} bytes",
113
+ source.get("media_type").and_then(|m| m.as_str()).unwrap_or("image"),
114
+ decoded_len(source.get("data").and_then(|d| d.as_str()).unwrap_or(""))
115
+ ),
116
+ }
117
+ }
118
+
119
+ /// Lift every image marker out of a tool result.
120
+ ///
121
+ /// Returns the result with each marker replaced by a placeholder string, and
122
+ /// the images as canonical blocks in the order they were found. An invalid
123
+ /// marker, or one past `max`, becomes a placeholder saying why — a tool that
124
+ /// returns a bad frame should not fail the run.
125
+ pub fn extract(value: &Value, max: usize) -> (Value, Vec<Value>) {
126
+ let mut images = Vec::new();
127
+ let text = walk(value, &mut |marker| {
128
+ if images.len() >= max {
129
+ return format!("[image omitted: at most {max} images per tool result]");
130
+ }
131
+ match image_block(marker) {
132
+ Ok(block) => {
133
+ let line = format!("[image {}: {} — attached]", images.len() + 1, describe(&block));
134
+ images.push(block);
135
+ line
136
+ }
137
+ Err(e) => format!("[image omitted: {e}]"),
138
+ }
139
+ });
140
+ (text, images)
141
+ }
142
+
143
+ /// The result as a step record or structured output keeps it: every marker's
144
+ /// payload replaced by its size, so megabytes of base64 do not land in the
145
+ /// response body, the audit log or session memory twice.
146
+ pub fn redact(value: &Value) -> Value {
147
+ walk(value, &mut |marker| {
148
+ let spec = marker.get(IMAGE_KEY).unwrap_or(&Value::Null);
149
+ match spec.get("data").and_then(|d| d.as_str()) {
150
+ Some(data) => json!({IMAGE_KEY: {
151
+ "media_type": spec.get("media_type").cloned().unwrap_or(Value::Null),
152
+ "bytes": decoded_len(data),
153
+ }}),
154
+ None => marker.clone(),
155
+ }
156
+ })
157
+ }
158
+
159
+ /// True when the value holds at least one image marker.
160
+ pub fn contains_images(value: &Value) -> bool {
161
+ match value {
162
+ v if is_marker(v) => true,
163
+ Value::Array(items) => items.iter().any(contains_images),
164
+ Value::Object(map) => map.values().any(contains_images),
165
+ _ => false,
166
+ }
167
+ }
168
+
169
+ fn walk<T: Into<Value>>(value: &Value, on_marker: &mut impl FnMut(&Value) -> T) -> Value {
170
+ match value {
171
+ v if is_marker(v) => on_marker(v).into(),
172
+ Value::Array(items) => Value::Array(items.iter().map(|v| walk(v, on_marker)).collect()),
173
+ Value::Object(map) => Value::Object(
174
+ map.iter().map(|(k, v)| (k.clone(), walk(v, on_marker))).collect(),
175
+ ),
176
+ other => other.clone(),
177
+ }
178
+ }
179
+
180
+ /// Keep only the `keep` most recent images in a conversation; older ones
181
+ /// become a line of text. Returns how many were dropped.
182
+ ///
183
+ /// Message structure is untouched — a `tool_result` keeps its id and stays
184
+ /// paired with its `tool_use` — only image blocks are swapped for text.
185
+ pub fn prune(messages: &mut [crate::agent::Message], keep: usize) -> usize {
186
+ let total: usize = messages.iter().map(|m| count(&m.content)).sum();
187
+ if total <= keep {
188
+ return 0;
189
+ }
190
+ let mut to_drop = total - keep;
191
+ let dropped = to_drop;
192
+ for m in messages.iter_mut() {
193
+ if to_drop == 0 {
194
+ break;
195
+ }
196
+ drop_oldest(&mut m.content, &mut to_drop);
197
+ }
198
+ dropped
199
+ }
200
+
201
+ /// Image blocks in one message's content, including inside tool results.
202
+ pub fn count(content: &Value) -> usize {
203
+ match content {
204
+ Value::Array(blocks) => blocks
205
+ .iter()
206
+ .map(|b| match b.get("type").and_then(|t| t.as_str()) {
207
+ Some("image") => 1,
208
+ Some("tool_result") => b.get("content").map(count).unwrap_or(0),
209
+ _ => 0,
210
+ })
211
+ .sum(),
212
+ _ => 0,
213
+ }
214
+ }
215
+
216
+ fn drop_oldest(content: &mut Value, to_drop: &mut usize) {
217
+ let Value::Array(blocks) = content else { return };
218
+ for b in blocks.iter_mut() {
219
+ if *to_drop == 0 {
220
+ return;
221
+ }
222
+ match b.get("type").and_then(|t| t.as_str()) {
223
+ Some("image") => {
224
+ *b = json!({"type": "text", "text": format!(
225
+ "[earlier image ({}) dropped from context to save tokens]",
226
+ describe(b)
227
+ )});
228
+ *to_drop -= 1;
229
+ }
230
+ Some("tool_result") => {
231
+ if let Some(inner) = b.get_mut("content") {
232
+ drop_oldest(inner, to_drop);
233
+ }
234
+ }
235
+ _ => {}
236
+ }
237
+ }
238
+ }
239
+
240
+ /// A canonical image block as an OpenAI `image_url` content part.
241
+ pub fn openai_part(block: &Value) -> Option<Value> {
242
+ let source = block.get("source")?;
243
+ let url = match source.get("type").and_then(|t| t.as_str()) {
244
+ Some("url") => source.get("url")?.as_str()?.to_string(),
245
+ Some("base64") => format!(
246
+ "data:{};base64,{}",
247
+ source.get("media_type")?.as_str()?,
248
+ source.get("data")?.as_str()?
249
+ ),
250
+ _ => return None,
251
+ };
252
+ Some(json!({"type": "image_url", "image_url": {"url": url}}))
253
+ }
254
+
255
+ #[cfg(test)]
256
+ mod tests {
257
+ use super::*;
258
+ use crate::agent::Message;
259
+
260
+ // A 1x1 transparent PNG.
261
+ const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
262
+
263
+ fn marker() -> Value {
264
+ json!({"$image": {"media_type": "image/png", "data": PNG}})
265
+ }
266
+
267
+ #[test]
268
+ fn a_marker_becomes_a_canonical_block() {
269
+ let block = image_block(&marker()).unwrap();
270
+ assert_eq!(block["type"], "image");
271
+ assert_eq!(block["source"]["type"], "base64");
272
+ assert_eq!(block["source"]["media_type"], "image/png");
273
+ }
274
+
275
+ #[test]
276
+ fn bad_images_are_refused_with_a_reason() {
277
+ assert!(image_block(&json!({"media_type": "image/bmp", "data": PNG})).unwrap_err().contains("unsupported"));
278
+ assert!(image_block(&json!({"media_type": "image/png", "data": "!!"})).unwrap_err().contains("base64"));
279
+ assert!(image_block(&json!({"url": "file:///etc/passwd"})).unwrap_err().contains("http"));
280
+ assert!(image_block(&json!({"data": PNG})).unwrap_err().contains("media_type"));
281
+ }
282
+
283
+ #[test]
284
+ fn urls_pass_through() {
285
+ let block = image_block(&json!({"url": "https://example.com/a.jpg"})).unwrap();
286
+ assert_eq!(block["source"]["url"], "https://example.com/a.jpg");
287
+ assert_eq!(openai_part(&block).unwrap()["image_url"]["url"], "https://example.com/a.jpg");
288
+ }
289
+
290
+ #[test]
291
+ fn extract_lifts_nested_markers_and_leaves_placeholders() {
292
+ let result = json!({"frame": marker(), "objects": [{"label": "bolt", "crop": marker()}], "n": 1});
293
+ let (text, images) = extract(&result, 8);
294
+ assert_eq!(images.len(), 2);
295
+ assert!(text["frame"].as_str().unwrap().starts_with("[image 1: image/png"));
296
+ assert!(text["objects"][0]["crop"].as_str().unwrap().starts_with("[image 2:"));
297
+ assert_eq!(text["n"], 1);
298
+ assert!(!text.to_string().contains(PNG));
299
+ }
300
+
301
+ #[test]
302
+ fn extract_respects_its_limit_and_survives_a_bad_marker() {
303
+ let result = json!([marker(), {"$image": {"media_type": "image/png", "data": "%%"}}, marker(), marker()]);
304
+ let (text, images) = extract(&result, 2);
305
+ assert_eq!(images.len(), 2);
306
+ assert!(text[1].as_str().unwrap().contains("not valid base64"));
307
+ assert!(text[2].as_str().unwrap().starts_with("[image 2:"));
308
+ assert!(text[3].as_str().unwrap().contains("at most 2"));
309
+ }
310
+
311
+ #[test]
312
+ fn redact_keeps_the_size_not_the_pixels() {
313
+ let r = redact(&json!({"frame": marker()}));
314
+ assert_eq!(r["frame"]["$image"]["media_type"], "image/png");
315
+ assert!(r["frame"]["$image"]["bytes"].as_u64().unwrap() > 0);
316
+ assert!(r["frame"]["$image"].get("data").is_none());
317
+ assert!(contains_images(&json!({"a": [marker()]})));
318
+ assert!(!contains_images(&json!({"a": [1, "x"]})));
319
+ }
320
+
321
+ #[test]
322
+ fn prune_drops_the_oldest_images_first() {
323
+ let img = image_block(&marker()).unwrap();
324
+ let mut messages = vec![
325
+ Message { role: "user".into(), content: json!([img.clone(), {"type": "text", "text": "look"}]) },
326
+ Message { role: "assistant".into(), content: json!([{"type": "tool_use", "id": "t1", "name": "cam", "input": {}}]) },
327
+ Message { role: "user".into(), content: json!([{"type": "tool_result", "tool_use_id": "t1",
328
+ "content": [{"type": "text", "text": "frame"}, img.clone()], "is_error": false}]) },
329
+ Message { role: "user".into(), content: json!([img.clone()]) },
330
+ ];
331
+ assert_eq!(prune(&mut messages, 1), 2);
332
+ assert_eq!(messages.iter().map(|m| count(&m.content)).sum::<usize>(), 1);
333
+ assert!(messages[0].content[0]["text"].as_str().unwrap().contains("dropped from context"));
334
+ assert_eq!(messages[2].content[0]["tool_use_id"], "t1", "pairing kept");
335
+ assert_eq!(count(&messages[3].content), 1, "the newest survives");
336
+ assert_eq!(prune(&mut messages, 1), 0);
337
+ }
338
+
339
+ #[test]
340
+ fn input_images_are_validated_and_capped() {
341
+ assert!(input_images(None).unwrap().is_empty());
342
+ assert_eq!(input_images(Some(&json!([marker(), {"url": "https://x.test/a.png"}]))).unwrap().len(), 2);
343
+ assert!(input_images(Some(&json!("nope"))).is_err());
344
+ let many = Value::Array(vec![marker(); MAX_IMAGES + 1]);
345
+ assert!(input_images(Some(&many)).is_err());
346
+ assert!(input_images(Some(&json!([{"url": "ftp://x"}]))).unwrap_err().starts_with("images[0]"));
347
+ }
348
+ }